{"meta":{"date":"2026-04-07","timezone":"UTC","generated_at":"2026-08-30T10:08:03Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":6,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T04:05:55Z"},"failed_at":["e2e","unknown"],"failed_test_count":8,"occurrences":[{"row_id":"271433c8095823e6c808b9397a69fbea1d23d0521ba75584bfc0f0a8044a3a79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","failed_at":"e2e","signature_id":"bc16479542f5518f199cdf90d49056dbd514a7388517335dffc90cafedff085b","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775539380 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 05:23:01.232\n \"ts\"=\"2026-04-07 05:23:01.232866\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-tczjgc-tp2rpx\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 05:23:05.133\n STEP: creating customer resources @ 04/07/26 05:23:05.225\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:23:50.322\n \"ts\"=\"2026-04-07 05:23:50.322296\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-tczjgc\" \"resourceGroup\"=\"rg-np-version-upgrade-tczjgc-tp2rpx\"\n STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:34:54.824\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775539380 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 05:23:01.232\n \"ts\"=\"2026-04-07 05:23:01.232866\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-tczjgc-tp2rpx\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 05:23:05.133\n STEP: creating customer resources @ 04/07/26 05:23:05.225\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:23:50.322\n \"ts\"=\"2026-04-07 05:23:50.322296\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-tczjgc\" \"resourceGroup\"=\"rg-np-version-upgrade-tczjgc-tp2rpx\"\n STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:34:54.824\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"3b6eb5a59a2b450ea066123e84c4116f2de0d1b423bae29b606790fdc3fbbf57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"3ec1c250b1873f16e4269fed3f407790014f8e73f8905463d224b674bd9cfcbd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:45]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:45]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"4d6f4e59d56f115e4966552166a27fbcf184145919908f38502f1f41b3a105f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","failed_at":"e2e","signature_id":"4f4470447da4e0eec3b1263376a6f4e4eabd73562a0c2787841c7a1fb3de6fef","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775539371 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/07/26 05:23:00.777\n \"ts\"=\"2026-04-07 05:23:00.777205\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9c578f-xr8n8x\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 05:23:04.384\n STEP: creating customer resources @ 04/07/26 05:23:04.465\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:23:51.414\n \"ts\"=\"2026-04-07 05:23:51.414251\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-9c578f\" \"resourceGroup\"=\"rg-np-version-upgrade-9c578f-xr8n8x\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:34:55.888\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/07/26 05:45:52.841\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/07/26 05:48:21.331\n STEP: capturing node release images before upgrade @ 04/07/26 05:48:21.463\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/07/26 05:48:21.503\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/07/26 05:49:13.715\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775539371 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/07/26 05:23:00.777\n \"ts\"=\"2026-04-07 05:23:00.777205\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9c578f-xr8n8x\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 05:23:04.384\n STEP: creating customer resources @ 04/07/26 05:23:04.465\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:23:51.414\n \"ts\"=\"2026-04-07 05:23:51.414251\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-9c578f\" \"resourceGroup\"=\"rg-np-version-upgrade-9c578f-xr8n8x\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:34:55.888\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/07/26 05:45:52.841\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/07/26 05:48:21.331\n STEP: capturing node release images before upgrade @ 04/07/26 05:48:21.463\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/07/26 05:48:21.503\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/07/26 05:49:13.715\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"b73cbb6b2892e9b78f78b5505a67053bad4cd1c1ffb47ed94c6f5fbca18c14c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","failed_at":"e2e","signature_id":"342fd958a98f57fdbc69171f73b9979d6370fb082d12515c8e5436854e410338","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"04/07/26 05:38:04.367\n STEP: attempting to update nodepool version to higher than cluster version @ 04/07/26 05:48:19.896\n \"ts\"=\"2026-04-07 05:48:20.460472\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l57kvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n \\\"target\\\": \\\"properties.version.id\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n STEP: attempting to update immutable platform profile fields @ 04/07/26 05:48:20.46\n \"ts\"=\"2026-04-07 05:48:21.151220\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l57kvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"4 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n \"ts\"=\"2026-04-07 05:48:21.271777\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 05:48:21.271880\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-rv6wgd\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n \"ts\"=\"2026-04-07 05:48:21.346235\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-rv6wgd\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 05:48:22.942477\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-07 05:48:22.942620\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-07 05:48:22.942678\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 05:48:22.944144\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\"\n \"ts\"=\"2026-04-07 05:48:22.944161\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n \"ts\"=\"2026-04-07 05:48:23.204714\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n \"ts\"=\"2026-04-07 05:48:23.305309\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\" \"deployment\"=\"customer-infra-negative-tests-cluster-rv6wgd\"\n \"ts\"=\"2026-04-07 05:48:23.320326\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 05:48:23.825441\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\" \"deployment\"=\"msi-deployment-13\"\n \"ts\"=\"2026-04-07 05:48:24.341336\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 05:48:24.341454\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 05:48:24.341626\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n \"ts\"=\"2026-04-07 06:07:46.163294\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n \"ts\"=\"2026-04-07 06:07:46.163320\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n04/07/26 05:38:04.367\n STEP: attempting to update nodepool version to higher than cluster version @ 04/07/26 05:48:19.896\n \"ts\"=\"2026-04-07 05:48:20.460472\" \"msg\"=\"invalid node pool version update\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l57kvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidRequestContent\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Invalid value: \\\\\\\"4.21.0\\\\\\\": invalid node pool version 4.21.0: cannot exceed control plane version 4.20.16\\\",\\n \\\"target\\\": \\\"properties.version.id\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n STEP: attempting to update immutable platform profile fields @ 04/07/26 05:48:20.46\n \"ts\"=\"2026-04-07 05:48:21.151220\" \"msg\"=\"updating immutable fields\" \"error\"=\"PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l57kvq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/negative-tests-cluster/nodePools/np-1\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: MultipleErrorsOccurred\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"MultipleErrorsOccurred\\\",\\n \\\"message\\\": \\\"4 errors occurred\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.vmSize\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.osDisk\\\"\\n },\\n {\\n \\\"code\\\": \\\"InvalidRequestContent\\\",\\n \\\"message\\\": \\\"Forbidden: field is immutable\\\",\\n \\\"target\\\": \\\"properties.platform.availabilityZone\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n \"ts\"=\"2026-04-07 05:48:21.271777\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 05:48:21.271880\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-negative-tests-cluster-rv6wgd\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n \"ts\"=\"2026-04-07 05:48:21.346235\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-negative-tests-cluster-rv6wgd\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 05:48:22.942477\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-07 05:48:22.942620\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-92ac8e02d20de6cf37d354d7e01d22da1b3bf7e3835a3ed2f6f6742a237a853f.yaml.gz\"\n \"ts\"=\"2026-04-07 05:48:22.942678\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 05:48:22.944144\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\"\n \"ts\"=\"2026-04-07 05:48:22.944161\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n \"ts\"=\"2026-04-07 05:48:23.204714\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\" \"deployment\"=\"pooledNonMsiScopedAssignments-oeayaalhj7wjy\"\n \"ts\"=\"2026-04-07 05:48:23.305309\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\" \"deployment\"=\"customer-infra-negative-tests-cluster-rv6wgd\"\n \"ts\"=\"2026-04-07 05:48:23.320326\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 05:48:23.825441\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-13\" \"deployment\"=\"msi-deployment-13\"\n \"ts\"=\"2026-04-07 05:48:24.341336\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 05:48:24.341454\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 05:48:24.341626\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n \"ts\"=\"2026-04-07 06:07:46.163294\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\n \"ts\"=\"2026-04-07 06:07:46.163320\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-negative-tests-l57kvq\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"dd9112acd0ae948718eb5970416b600373290da45ffa304e9378190b35a2be31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:51]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:51]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"f2970f697773e368940262ebd515ea85333007b6c1d5087707a3e97da1dd3b8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","failed_at":"e2e","signature_id":"ccbd3aa0996ca39ea8f78950f49effcec35d7962ca56c70dc03209ef9115be87","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775539368 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/07/26 05:23:00.767\n \"ts\"=\"2026-04-07 05:23:00.767797\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-mbhzml\"\n STEP: creating cluster parameters @ 04/07/26 05:23:05.374\n STEP: creating customer resources @ 04/07/26 05:23:05.374\n STEP: creating the HCP cluster @ 04/07/26 05:24:00.793\n \"ts\"=\"2026-04-07 05:24:00.794080\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-mbhzml\"\n STEP: getting credentials @ 04/07/26 05:36:13.695\n STEP: creating the node pool with initial labels and taints @ 04/07/26 05:37:48.46\n STEP: waiting for nodes to be ready @ 04/07/26 05:49:58.944\n STEP: verifying initial labels are present on nodes @ 04/07/26 05:49:59.105\n STEP: verifying initial taints are present on nodes @ 04/07/26 05:49:59.128\n STEP: updating nodepool with new taints and scaling up @ 04/07/26 05:49:59.128\n STEP: verifying nodes are scaled to the expected count @ 04/07/26 05:59:46.317\n STEP: verifying new taint is present on newly created node @ 04/07/26 05:59:46.477\n STEP: updating nodepool with a new label and scaling up @ 04/07/26 05:59:46.499\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775539368 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/07/26 05:23:00.767\n \"ts\"=\"2026-04-07 05:23:00.767797\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-mbhzml\"\n STEP: creating cluster parameters @ 04/07/26 05:23:05.374\n STEP: creating customer resources @ 04/07/26 05:23:05.374\n STEP: creating the HCP cluster @ 04/07/26 05:24:00.793\n \"ts\"=\"2026-04-07 05:24:00.794080\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-mbhzml\"\n STEP: getting credentials @ 04/07/26 05:36:13.695\n STEP: creating the node pool with initial labels and taints @ 04/07/26 05:37:48.46\n STEP: waiting for nodes to be ready @ 04/07/26 05:49:58.944\n STEP: verifying initial labels are present on nodes @ 04/07/26 05:49:59.105\n STEP: verifying initial taints are present on nodes @ 04/07/26 05:49:59.128\n STEP: updating nodepool with new taints and scaling up @ 04/07/26 05:49:59.128\n STEP: verifying nodes are scaled to the expected count @ 04/07/26 05:59:46.317\n STEP: verifying new taint is present on newly created node @ 04/07/26 05:59:46.477\n STEP: updating nodepool with a new label and scaling up @ 04/07/26 05:59:46.499\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"ff8af2825e0b10f3a8cc74e415bc097717adcbfc99a24f51c8753c9e13530005","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-07T06:08:14Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T04:05:16Z"},"failed_at":["e2e","unknown"],"failed_test_count":4,"occurrences":[{"row_id":"223a20713d600a9a8b04f681c7d5c8c2268c7ac89f1b884eb0134ff6861c2985","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","failed_at":"e2e","signature_id":"500d409f3d8fb29df0602cb09997135f46bcf75fd9f7fb14e42d1ce74f68449c","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"l run any cleanup and reporting nodes but will skip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Authorized CIDRs Connectivity should allow API access only from authorized VM IP (Spec Runtime: 1h29m59.887s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:45\n In [It] (Node Runtime: 1h29m59.887s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:45\n At [By Step] updating cluster to remove VM from authorized CIDRs (Step Runtime: 1m1.446s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:392\n\n Spec Goroutine\n goroutine 123 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x7f1a4e242a60, 0xc000f141e0}, 0x81dda58?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8cebf60, {0x7f1a4e242a60, 0xc000f141e0}, 0xc0016d0cf0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func5.1.1({0x7f1a4e242a60, 0xc000f141e0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:417\n | Expect(err).NotTo(HaveOccurred())\n | \n \u003e _, err = poller.PollUntilDone(ctx, nil)\n | Expect(err).NotTo(HaveOccurred())\n | \n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc8d40?, 0xc000f141e0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 121\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:418 @ 04/07/26 05:37:42.895\n \"ts\"=\"2026-04-07 05:37:42.895340\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 05:37:42.895411\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cidr-connectivity-test-fkzk9c\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\"\n \"ts\"=\"2026-04-07 05:37:43.640257\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cidr-connectivity-test-fkzk9c\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 05:37:45.800087\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"test-vm\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\"\n \"ts\"=\"2026-04-07 05:37:46.577328\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-5585d0232685e8d5bbce4642dbb894d73a2597e55ff59b2a6006f2460a217da1.yaml.gz\"\n \"ts\"=\"2026-04-07 05:37:46.577436\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-5585d0232685e8d5bbce4642dbb894d73a2597e55ff59b2a6006f2460a217da1.yaml.gz\"\n \"ts\"=\"2026-04-07 05:37:46.577508\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 05:37:46.578979\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-80\"\n \"ts\"=\"2026-04-07 05:37:46.579001\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\"\n \"ts\"=\"2026-04-07 05:37:46.885972\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-80\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 05:37:47.023444\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-80\" \"deployment\"=\"msi-deployment-80\"\n \"ts\"=\"2026-04-07 05:37:47.068942\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\" \"deployment\"=\"test-vm\"\n \"ts\"=\"2026-04-07 05:37:47.818656\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\" \"deployment\"=\"pooledNonMsiScopedAssignments-wihoigd7cpguy\"\n \"ts\"=\"2026-04-07 05:37:48.556118\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\" \"deployment\"=\"customer-infra-cidr-connectivity-test-fkzk9c\"\n \"ts\"=\"2026-04-07 05:37:49.295808\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 05:37:49.295900\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 05:37:49.296042\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nl run any cleanup and reporting nodes but will skip all remaining specs. Interrupt again to skip cleanup.\n Here's a current progress report:\n Authorized CIDRs Connectivity should allow API access only from authorized VM IP (Spec Runtime: 1h29m59.887s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:45\n In [It] (Node Runtime: 1h29m59.887s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:45\n At [By Step] updating cluster to remove VM from authorized CIDRs (Step Runtime: 1m1.446s)\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:392\n\n Spec Goroutine\n goroutine 123 [select]\n github.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x7f1a4e242a60, 0xc000f141e0}, 0x81dda58?)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36\n github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8cebf60, {0x7f1a4e242a60, 0xc000f141e0}, 0xc0016d0cf0)\n /go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284\n \u003e github.com/Azure/ARO-HCP/test/e2e.init.func5.1.1({0x7f1a4e242a60, 0xc000f141e0})\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:417\n | Expect(err).NotTo(HaveOccurred())\n | \n \u003e _, err = poller.PollUntilDone(ctx, nil)\n | Expect(err).NotTo(HaveOccurred())\n | \n github.com/onsi/ginkgo/v2/internal.extractBodyFunction.func2({0x8cc8d40?, 0xc000f141e0?})\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/node.go:465\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901\n github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 121\n /go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888\n ------------------------------\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:418 @ 04/07/26 05:37:42.895\n \"ts\"=\"2026-04-07 05:37:42.895340\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 05:37:42.895411\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cidr-connectivity-test-fkzk9c\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\"\n \"ts\"=\"2026-04-07 05:37:43.640257\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cidr-connectivity-test-fkzk9c\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 05:37:45.800087\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"test-vm\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\"\n \"ts\"=\"2026-04-07 05:37:46.577328\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-5585d0232685e8d5bbce4642dbb894d73a2597e55ff59b2a6006f2460a217da1.yaml.gz\"\n \"ts\"=\"2026-04-07 05:37:46.577436\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-5585d0232685e8d5bbce4642dbb894d73a2597e55ff59b2a6006f2460a217da1.yaml.gz\"\n \"ts\"=\"2026-04-07 05:37:46.577508\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 05:37:46.578979\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-80\"\n \"ts\"=\"2026-04-07 05:37:46.579001\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\"\n \"ts\"=\"2026-04-07 05:37:46.885972\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-80\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 05:37:47.023444\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-80\" \"deployment\"=\"msi-deployment-80\"\n \"ts\"=\"2026-04-07 05:37:47.068942\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\" \"deployment\"=\"test-vm\"\n \"ts\"=\"2026-04-07 05:37:47.818656\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\" \"deployment\"=\"pooledNonMsiScopedAssignments-wihoigd7cpguy\"\n \"ts\"=\"2026-04-07 05:37:48.556118\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\" \"deployment\"=\"customer-infra-cidr-connectivity-test-fkzk9c\"\n \"ts\"=\"2026-04-07 05:37:49.295808\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 05:37:49.295900\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 05:37:49.296042\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"e2e-cidr-connectivity-mbl5kf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"2a477ea2142a7303ee284f7e4e93bc90f1a36361d3c86becf2fa3862539839be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","failed_at":"e2e","signature_id":"80425a41a776bc9c054fe74e89b94fff4fb1f63049b7a42543353153422bbf16","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"rnetes.io/os\"=\"linux\" \"kubernetes.io/hostname\"=\"p2h5q7b0k0o4w8k-np-1-lh727-jwtwb\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/zone\"=\"1\" \"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"brazilsouth\" \"kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/region\"=\"brazilsouth\" \"hypershift.openshift.io/nodePool\"=\"p2h5q7b0k0o4w8k-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"hypershift.openshift.io/managed\"=\"true\" \"key1\"=\"value1\" \"kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-07 05:58:53.380998\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"p2h5q7b0k0o4w8k-np-1-lh727-mn9jj\" \"labels\"={\"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"node-role.kubernetes.io/worker\"=\"\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/hostname\"=\"p2h5q7b0k0o4w8k-np-1-lh727-mn9jj\" \"node.openshift.io/os_id\"=\"rhel\" \"hypershift.openshift.io/nodePool\"=\"p2h5q7b0k0o4w8k-np-1\" \"key1\"=\"value1\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"brazilsouth\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"brazilsouth\" \"kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-07 05:58:53.381045\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"p2h5q7b0k0o4w8k-np-1-lh727-nvdvx\" \"labels\"={\"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"brazilsouth\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"key2\"=\"value2\" \"node.openshift.io/os_id\"=\"rhel\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/nodePool\"=\"p2h5q7b0k0o4w8k-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/hostname\"=\"p2h5q7b0k0o4w8k-np-1-lh727-nvdvx\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/region\"=\"brazilsouth\" \"topology.kubernetes.io/zone\"=\"0\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n STEP: verifying original labels persist on 3 nodes @ 04/07/26 05:58:53.381\n STEP: verifying original taints persist on 2 nodes @ 04/07/26 05:58:53.381\n \"ts\"=\"2026-04-07 05:58:53.381321\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 05:58:53.381397\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-9hptlr\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\"\n \"ts\"=\"2026-04-07 05:58:54.127914\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-9hptlr\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 05:58:55.905622\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-07 05:58:55.905719\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-07 05:58:55.905778\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 05:58:55.907347\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-74\"\n \"ts\"=\"2026-04-07 05:58:55.907358\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\"\n \"ts\"=\"2026-04-07 05:58:56.059340\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-74\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 05:58:56.342319\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-74\" \"deployment\"=\"msi-deployment-74\"\n \"ts\"=\"2026-04-07 05:58:56.377890\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-07 05:58:57.390414\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-9hptlr\"\n \"ts\"=\"2026-04-07 05:58:58.337339\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 05:58:58.337474\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 05:58:58.337604\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nrnetes.io/os\"=\"linux\" \"kubernetes.io/hostname\"=\"p2h5q7b0k0o4w8k-np-1-lh727-jwtwb\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/zone\"=\"1\" \"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"brazilsouth\" \"kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/region\"=\"brazilsouth\" \"hypershift.openshift.io/nodePool\"=\"p2h5q7b0k0o4w8k-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"hypershift.openshift.io/managed\"=\"true\" \"key1\"=\"value1\" \"kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-07 05:58:53.380998\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"p2h5q7b0k0o4w8k-np-1-lh727-mn9jj\" \"labels\"={\"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"node-role.kubernetes.io/worker\"=\"\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/hostname\"=\"p2h5q7b0k0o4w8k-np-1-lh727-mn9jj\" \"node.openshift.io/os_id\"=\"rhel\" \"hypershift.openshift.io/nodePool\"=\"p2h5q7b0k0o4w8k-np-1\" \"key1\"=\"value1\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"brazilsouth\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"brazilsouth\" \"kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-07 05:58:53.381045\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"p2h5q7b0k0o4w8k-np-1-lh727-nvdvx\" \"labels\"={\"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"brazilsouth\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"key2\"=\"value2\" \"node.openshift.io/os_id\"=\"rhel\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/nodePool\"=\"p2h5q7b0k0o4w8k-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/hostname\"=\"p2h5q7b0k0o4w8k-np-1-lh727-nvdvx\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/os\"=\"linux\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/region\"=\"brazilsouth\" \"topology.kubernetes.io/zone\"=\"0\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n STEP: verifying original labels persist on 3 nodes @ 04/07/26 05:58:53.381\n STEP: verifying original taints persist on 2 nodes @ 04/07/26 05:58:53.381\n \"ts\"=\"2026-04-07 05:58:53.381321\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 05:58:53.381397\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-9hptlr\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\"\n \"ts\"=\"2026-04-07 05:58:54.127914\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-9hptlr\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 05:58:55.905622\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-07 05:58:55.905719\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-07 05:58:55.905778\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 05:58:55.907347\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-74\"\n \"ts\"=\"2026-04-07 05:58:55.907358\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\"\n \"ts\"=\"2026-04-07 05:58:56.059340\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-74\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 05:58:56.342319\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-74\" \"deployment\"=\"msi-deployment-74\"\n \"ts\"=\"2026-04-07 05:58:56.377890\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-07 05:58:57.390414\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-9hptlr\"\n \"ts\"=\"2026-04-07 05:58:58.337339\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 05:58:58.337474\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 05:58:58.337604\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-fvhpb2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"6bdd972ecd177b0bbeaaa4ad863299562d80a81a83fc5648544a8cbce0f6c2ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","failed_at":"e2e","signature_id":"f15184542c39bb774c3be8e478ae04eeb6b9843c71ff6dda60d4522220a27a0f","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000958690\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-pmp787\" in resourcegroup=\"rg-np-version-upgrade-pmp787-kwgkwl\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001248460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl\",\n },\n \u003c*fmt.wrapError | 0xc001104340\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000958690\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-pmp787\" in resourcegroup=\"rg-np-version-upgrade-pmp787-kwgkwl\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001248460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl\",\n },\n \u003c*fmt.wrapError | 0xc001104340\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded"}},{"row_id":"e6b5c0237b7bb7b718e9499e7b839a41bda02ea7c1ebbb084cbb277aa93c59bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-07T06:07:36Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T04:04:40Z"},"failed_at":["e2e","unknown"],"failed_test_count":4,"occurrences":[{"row_id":"42e57506f7004d7665f23d241270ec09dcb5e7ffce1ac0c69ac8e2bd4b39a598","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-07T06:07:00Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"81f3b95d66c88a5c3c8979bf483530f0488a19889531a48b6559fae0d095493f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"853e299216f9117fbabdbec3858e0deb6d715f2cb4fbba56bd4cff169c7e37a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","failed_at":"e2e","signature_id":"771a9b202a2b5fcd9bc2f6284c90ce9ab0922fc86ee71395d36f43a8a781d0d3","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775538149 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 05:02:29.603\n \"ts\"=\"2026-04-07 05:02:29.603820\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 05:02:33.781\n STEP: creating customer resources @ 04/07/26 05:02:33.862\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:03:26.183\n \"ts\"=\"2026-04-07 05:03:26.183582\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-9mxh7s\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\n STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:19:16.826\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154 @ 04/07/26 06:04:16.833\n \"ts\"=\"2026-04-07 06:04:16.834185\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 06:04:16.834270\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-9mxh7s-2vp6d7\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\n \"ts\"=\"2026-04-07 06:04:17.366829\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-9mxh7s-2vp6d7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 06:04:18.997857\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-99031ad0c929f4b469b552900cdb42a4d2445ad8090c3f5e30a3cba67f4c8788.yaml.gz\"\n \"ts\"=\"2026-04-07 06:04:18.997955\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-99031ad0c929f4b469b552900cdb42a4d2445ad8090c3f5e30a3cba67f4c8788.yaml.gz\"\n \"ts\"=\"2026-04-07 06:04:18.998020\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 06:04:18.999596\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-1\"\n \"ts\"=\"2026-04-07 06:04:18.999607\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\n \"ts\"=\"2026-04-07 06:04:19.153005\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-1\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 06:04:19.305470\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-1\" \"deployment\"=\"msi-deployment-1\"\n \"ts\"=\"2026-04-07 06:04:19.951537\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\" \"deployment\"=\"pooledNonMsiScopedAssignments-pr77ts5rpnbqu\"\n \"ts\"=\"2026-04-07 06:04:20.715265\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-9mxh7s-2vp6d7\"\n \"ts\"=\"2026-04-07 06:04:21.424419\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 06:04:21.424586\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 06:04:21.424739\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775538149 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 05:02:29.603\n \"ts\"=\"2026-04-07 05:02:29.603820\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 05:02:33.781\n STEP: creating customer resources @ 04/07/26 05:02:33.862\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 05:03:26.183\n \"ts\"=\"2026-04-07 05:03:26.183582\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-9mxh7s\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\n STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 05:19:16.826\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154 @ 04/07/26 06:04:16.833\n \"ts\"=\"2026-04-07 06:04:16.834185\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 06:04:16.834270\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-9mxh7s-2vp6d7\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\n \"ts\"=\"2026-04-07 06:04:17.366829\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-9mxh7s-2vp6d7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 06:04:18.997857\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-99031ad0c929f4b469b552900cdb42a4d2445ad8090c3f5e30a3cba67f4c8788.yaml.gz\"\n \"ts\"=\"2026-04-07 06:04:18.997955\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-99031ad0c929f4b469b552900cdb42a4d2445ad8090c3f5e30a3cba67f4c8788.yaml.gz\"\n \"ts\"=\"2026-04-07 06:04:18.998020\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 06:04:18.999596\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-1\"\n \"ts\"=\"2026-04-07 06:04:18.999607\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\n \"ts\"=\"2026-04-07 06:04:19.153005\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-1\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 06:04:19.305470\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-1\" \"deployment\"=\"msi-deployment-1\"\n \"ts\"=\"2026-04-07 06:04:19.951537\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\" \"deployment\"=\"pooledNonMsiScopedAssignments-pr77ts5rpnbqu\"\n \"ts\"=\"2026-04-07 06:04:20.715265\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-9mxh7s-2vp6d7\"\n \"ts\"=\"2026-04-07 06:04:21.424419\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 06:04:21.424586\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 06:04:21.424739\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-9mxh7s-dwkt6k\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"ba047d9338605c1e595dc7c7749fa0746937accd6b36d1bfb9af2200f3088caf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","failed_at":"e2e","signature_id":"46a267eec014b8e48c35e8603c9ead475405836ebd27b349d3b51b246bd00195","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f2c9e0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ztlcvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ztlcvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00178b8c0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"9df649c9-418f-4940-bec8-8c0e548e3db3\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Expires\": [\"-1\"],\n \"Date\": [\n \"Tue, 07 Apr 2026 05:20:56 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"b0981fa0-473c-423b-8940-929e8b4ed066\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"1246\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"WESTEUROPE:20260407T052056Z:9df649c9-418f-4940-bec8-8c0e548e3db3\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/westeurope/95c613cd-bdd4-496d-928d-a3fc6f2b736e\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 9189BB39768F4C5AA7879357F88C2139 Ref B: CHI261140501031 Ref C: 2026-04-07T05:20:55Z\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000f2c920\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000f2c9e0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ztlcvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-ztlcvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc00178b8c0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"9df649c9-418f-4940-bec8-8c0e548e3db3\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Expires\": [\"-1\"],\n \"Date\": [\n \"Tue, 07 Apr 2026 05:20:56 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"b0981fa0-473c-423b-8940-929e8b4ed066\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"1246\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"WESTEUROPE:20260407T052056Z:9df649c9-418f-4940-bec8-8c0e548e3db3\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/westeurope/95c613cd-bdd4-496d-928d-a3fc6f2b736e\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 9189BB39768F4C5AA7879357F88C2139 Ref B: CHI261140501031 Ref C: 2026-04-07T05:20:55Z\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000f2c920\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"m...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T00:23:16Z"},"failed_at":["e2e"],"failed_test_count":32,"occurrences":[{"row_id":"067dc5f1ea3f453f18de4bd136bc36da9fcc4f956643eb60b21851e9de2c70e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"545f07c5de94eb876fe5c60d1fd48aabe80ce9fe76291be55b34245f3cddfa0e","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526666 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\n STEP: creating resource group @ 04/07/26 01:51:11.527\n \"ts\"=\"2026-04-07 01:51:11.527298\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-hhmb8r-pnkvwg\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:51:16.274\n STEP: creating customer resources @ 04/07/26 01:51:16.408\n STEP: creating the HCP cluster with version 4.20.17 @ 04/07/26 01:52:26.096\n \"ts\"=\"2026-04-07 01:52:26.096671\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-hhmb8r\" \"resourceGroup\"=\"rg-np-version-upgrade-hhmb8r-pnkvwg\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526666 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\n STEP: creating resource group @ 04/07/26 01:51:11.527\n \"ts\"=\"2026-04-07 01:51:11.527298\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-hhmb8r-pnkvwg\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:51:16.274\n STEP: creating customer resources @ 04/07/26 01:51:16.408\n STEP: creating the HCP cluster with version 4.20.17 @ 04/07/26 01:52:26.096\n \"ts\"=\"2026-04-07 01:52:26.096671\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-hhmb8r\" \"resourceGroup\"=\"rg-np-version-upgrade-hhmb8r-pnkvwg\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"06831b61516f683b58bc8b8617bb5e46ff8221f69cc024693c7f892373c8f89f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"0eb17ac0fa73cbaf6bfd96b147f1e87899b4eb9122843f96122b57a919ecf965","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526682 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n STEP: creating a resource group @ 04/07/26 01:51:22.035\n \"ts\"=\"2026-04-07 01:51:22.035287\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-2qcm89\"\n STEP: creating cluster parameters @ 04/07/26 01:51:26.178\n STEP: creating customer resources @ 04/07/26 01:51:26.178\n STEP: creating the HCP cluster @ 04/07/26 01:52:43.309\n \"ts\"=\"2026-04-07 01:52:43.309255\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-2qcm89\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526682 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should not be able to perform various invalid operations on cluster resources [PreLaunchSetup:None, Positivity:Negative, Importance:Medium, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:41\n STEP: creating a resource group @ 04/07/26 01:51:22.035\n \"ts\"=\"2026-04-07 01:51:22.035287\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-negative-tests-2qcm89\"\n STEP: creating cluster parameters @ 04/07/26 01:51:26.178\n STEP: creating customer resources @ 04/07/26 01:51:26.178\n STEP: creating the HCP cluster @ 04/07/26 01:52:43.309\n \"ts\"=\"2026-04-07 01:52:43.309255\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"negative-tests-cluster\" \"resourceGroup\"=\"rg-negative-tests-2qcm89\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"13299ef54cb02d44c1c65a456c6a1d9c69fe8edf53c88c96778484cfb2851d95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"38826b533269d7a9bb238361448ad259ba2286646e96c6502152a76dcb1076fe","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014db050\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bqjn7p\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bqjn7p\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000614870\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014db050\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-bqjn7p\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-bqjn7p\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000614870\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-bqjn7p\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"1365f7d9f3af48c6b552152119479f65eb32b463fa76d242b3fd6735badfd0e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"49ed41086fe7a9ec2b601ce88c197476ebb461138a74a4df1c55c826a052fa0f","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526671 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to use workload identity via the cluster OIDC issuer URL [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:169\n STEP: creating a resource group @ 04/07/26 01:51:11.583\n \"ts\"=\"2026-04-07 01:51:11.583892\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"oidc-wi-2sj8tw\"\n STEP: creating cluster parameters @ 04/07/26 01:51:16.234\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/07/26 01:51:16.234\n STEP: creating the HCP cluster @ 04/07/26 01:52:38.669\n \"ts\"=\"2026-04-07 01:52:38.669995\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"oidc-wi-cluster\" \"resourceGroup\"=\"oidc-wi-2sj8tw\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526671 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to use workload identity via the cluster OIDC issuer URL [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:169\n STEP: creating a resource group @ 04/07/26 01:51:11.583\n \"ts\"=\"2026-04-07 01:51:11.583892\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"oidc-wi-2sj8tw\"\n STEP: creating cluster parameters @ 04/07/26 01:51:16.234\n STEP: creating customer resources (infrastructure and managed identities) for cluster @ 04/07/26 01:51:16.234\n STEP: creating the HCP cluster @ 04/07/26 01:52:38.669\n \"ts\"=\"2026-04-07 01:52:38.669995\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"oidc-wi-cluster\" \"resourceGroup\"=\"oidc-wi-2sj8tw\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"16dfa33ae40002b655e728f52f17e3e9826aa04c73875467f8351ac9eb78ef1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"cf73ef57a2796840467d871dd1126493528568bf6b577c2cbd568c2f109e27c8","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011aa540\u003e: \n failed to create HCP cluster ea-listw6gcm5, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: failed waiting for cluster=\"ea-listw6gcm5\" in resourcegroup=\"ea-list-4w9cgg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listw6gcm5, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: failed waiting for cluster=\\\"ea-listw6gcm5\\\" in resourcegroup=\\\"ea-list-4w9cgg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f6a6b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg\",\n },\n \u003c*fmt.wrapErrors | 0xc0011aa510\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listw6gcm5\\\" in resourcegroup=\\\"ea-list-4w9cgg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f6a6b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:84]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011aa540\u003e: \n failed to create HCP cluster ea-listw6gcm5, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: failed waiting for cluster=\"ea-listw6gcm5\" in resourcegroup=\"ea-list-4w9cgg\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-listw6gcm5, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: failed waiting for cluster=\\\"ea-listw6gcm5\\\" in resourcegroup=\\\"ea-list-4w9cgg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f6a6b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg\",\n },\n \u003c*fmt.wrapErrors | 0xc0011aa510\u003e{\n msg: \"failed waiting for cluster=\\\"ea-listw6gcm5\\\" in resourcegroup=\\\"ea-list-4w9cgg\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f6a6b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-listw6gcm5 in resource group ea-list-4w9cgg\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"1d23f3b09fc97de87fb27d182fb00a4dadf5de2aaa9b43bf14251bdd66768f42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"e3bf0e48b6a7ad4445c218645ad83a075e948fb4e051c71c17cf9d0ae2652300","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00150e3c0\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-hqc5zn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hqc5zn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-hqc5zn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hqc5zn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009ad190\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hqc5zn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00150e3c0\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-hqc5zn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hqc5zn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-hqc5zn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hqc5zn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009ad190\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-hqc5zn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"23373dceffd7e68fe25bf059f7c0f0c27c7d33f9bc55602c98f6890bb969f302","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"5a84ec606cf94e594e22c62e9fe85fd628da238028d88ecf1ca775367060da89","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001802780\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5m87gs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-5m87gs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e3b670\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs\",\n },\n \u003c*fmt.wrapErrors | 0xc001802750\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-5m87gs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e3b670\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001802780\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5m87gs\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-5m87gs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e3b670\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs\",\n },\n \u003c*fmt.wrapErrors | 0xc001802750\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-5m87gs\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e3b670\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-5m87gs\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"24846eae91e82a47fd3b23c46be6ea0ade3b87f4a31f2f706a9236a7f1a30bcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"490a38b6630e9d08af57dc89b154f250a304afaf6ce6fbdf3395ae1e55a94afd","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009ea5a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-gsft5q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009484b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q\",\n },\n \u003c*fmt.wrapErrors | 0xc0009ea570\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-gsft5q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009484b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009ea5a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-gsft5q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009484b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q\",\n },\n \u003c*fmt.wrapErrors | 0xc0009ea570\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-gsft5q\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009484b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"33e5bb67d5672bac66d2a67ef37311713ca9ec33ee91eb59b9ab97217506e3bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"325ed533ecfab9ad14356e1c3682e386e21134251bde23aff995a4190cec9728","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012d86f0\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001247440\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h\",\n },\n \u003c*fmt.wrapError | 0xc0005187e0\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012d86f0\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001247440\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h\",\n },\n \u003c*fmt.wrapError | 0xc0005187e0\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded"}},{"row_id":"347f6e347e5a094aed7b3c66c7f14ba66bf4b5053e4f0105e4dc18d00e39c51a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"34c8538cd1cf99db24364475fbc228e23e26d2362e2a83d8f0fdfb52ba4cf1f6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012d06c0\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-5875bd\" in resourcegroup=\"rg-stable-4-20-sx464q\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-5875bd\\\" in resourcegroup=\\\"rg-stable-4-20-sx464q\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001362380\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012d06c0\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-5875bd\" in resourcegroup=\"rg-stable-4-20-sx464q\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-5875bd\\\" in resourcegroup=\\\"rg-stable-4-20-sx464q\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001362380\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-5875bd in resource group rg-stable-4-20-sx464q\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"3fc9b68cd7da45c026325e470220ac10e68867d6c8a05d69a1ffe2543a5932d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"48d66cb903bed78ecc5b3870134f6236fde8da937c87322b0e8aea6adb477cb3","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00092ca20\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-98hkw2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-98hkw2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e839a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2\",\n },\n \u003c*fmt.wrapErrors | 0xc00092c9f0\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-98hkw2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e839a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:83]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00092ca20\u003e: \n failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-98hkw2\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster np-autoscale-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-98hkw2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e839a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2\",\n },\n \u003c*fmt.wrapErrors | 0xc00092c9f0\u003e{\n msg: \"failed waiting for cluster=\\\"np-autoscale-cluster\\\" in resourcegroup=\\\"np-autoscaling-98hkw2\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e839a0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster np-autoscale-cluster in resource group np-autoscaling-98hkw2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"404941b0ba558b46b8fe2a5b185cfb4eb6380d8b9c30c5549761966a36156961","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"3298511640d7e93d00a720ae5553b54712470ec9b619dda50523cf8dd9b743df","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014d1560\u003e: \n failed to create HCP cluster cluster-zstream-4-20-s2sjhd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: failed waiting for cluster=\"cluster-zstream-4-20-s2sjhd\" in resourcegroup=\"rg-zstream-upgrade-4-20-tjt8qk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-s2sjhd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: failed waiting for cluster=\\\"cluster-zstream-4-20-s2sjhd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tjt8qk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b6770\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk\",\n },\n \u003c*fmt.wrapErrors | 0xc0014d1530\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-s2sjhd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tjt8qk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b6770\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0014d1560\u003e: \n failed to create HCP cluster cluster-zstream-4-20-s2sjhd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: failed waiting for cluster=\"cluster-zstream-4-20-s2sjhd\" in resourcegroup=\"rg-zstream-upgrade-4-20-tjt8qk\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cluster-zstream-4-20-s2sjhd, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: failed waiting for cluster=\\\"cluster-zstream-4-20-s2sjhd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tjt8qk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b6770\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk\",\n },\n \u003c*fmt.wrapErrors | 0xc0014d1530\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-20-s2sjhd\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-tjt8qk\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0013b6770\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-20-s2sjhd in resource group rg-zstream-upgrade-4-20-tjt8qk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"4890a72a4ac68bcf5b6b5ad5f540e7ba4e44842ef9a7fddaddfbad908b40e122","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"58723eeca32480c2a7147989b14ad5770f17d9613a970f9c8d123ee9128db707","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:123]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011acae0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-dvhdz5, error: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dvhdz5\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-dvhdz5, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-dvhdz5\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014c5b20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-dvhdz5\",\n },\n \u003c*fmt.wrapError | 0xc001086580\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-dvhdz5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:123]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011acae0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-dvhdz5, error: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-dvhdz5\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-dvhdz5, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-dvhdz5\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014c5b20\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group tls-endpoint-cluster-dvhdz5\",\n },\n \u003c*fmt.wrapError | 0xc001086580\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"tls-endpoint-hcp-cluster\\\" in resourcegroup=\\\"tls-endpoint-cluster-dvhdz5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded"}},{"row_id":"4bb035f9e935ca6ddbb9fb390803fe56f2bd583307e98b08ad9a42c823db3994","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:34]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:34]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"4be8252d818ac1d3277ccd672ee6aea668adae36fa6391a494938015189980b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"19b8b89f21ef63dfdd459a5ae0dde3ec9e67b8586ab0532fdd1a4d72e5dd9677","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001193290\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-879vc5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-879vc5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001180090\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001193290\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-879vc5\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-879vc5\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001180090\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-879vc5\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"504453cf9d163020e0ee7ff489aec3f0a646e86251651ee0b5a250ab50f5e1ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"4e32fb5d7ee25f5f0a0bef0524e92b9066de0d63ef1fec9063e4350475d0eb88","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000bdb0e0\u003e: \n failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5b92nh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-5b92nh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e905f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh\",\n },\n \u003c*fmt.wrapErrors | 0xc000bdb0b0\u003e{\n msg: \"failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-5b92nh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e905f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:112]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000bdb0e0\u003e: \n failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-5b92nh\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ephemeral-disk, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-5b92nh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e905f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh\",\n },\n \u003c*fmt.wrapErrors | 0xc000bdb0b0\u003e{\n msg: \"failed waiting for cluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-5b92nh\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e905f0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ephemeral-disk in resource group ephemeral-osdisk-5b92nh\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"5120077ce709fe2fcf3dff034c405ab2da8cef811802c68f6ea487a55d21b40a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"f1e615836088d3a70d52b7c6f718a74bd4b874c9412a0b8713b7bb1e5df7bace","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"amd64.s:1693 +0x1 fp=0xc000bf1fe8 sp=0xc000bf1fe0 pc=0x2dbecc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 119 gp=0xc000bb48c0 m=nil [select]:\nruntime.gopark(0xc000e602f0?, 0x2?, 0xa8?, 0xf8?, 0xc000e602e4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e60170 sp=0xc000e60150 pc=0x2db650e\nruntime.selectgo(0xc000e602f0, 0xc000e602e0, 0xc000b8d940?, 0x0, 0x35cdd3c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000e602b0 sp=0xc000e60170 pc=0x2d912f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8cb0988, 0xc0009b4460}, 0x81dda58?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000e60320 sp=0xc000e602b0 pc=0x35cd496\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8cebf60, {0x8cb0988, 0xc0009b4460}, 0x0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x7b2 fp=0xc000e60668 sp=0xc000e60320 pc=0x3abad52\ngithub.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8cb09f8?, 0xc000340fc0?}, {{0x8cc8c68?, 0xc000638b80?}, 0xc00120a680?}, 0xc000e60b70, {0xc000ea4260, 0x1e}, {0xc000bc2440, 0x1b}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:694 +0x405 fp=0xc000e608e0 sp=0xc000e60668 pc=0x5261e85\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc00127d658, 0x7}, ...}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346 +0x497 fp=0xc000e60c40 sp=0xc000e608e0 pc=0x5255fd7\ngithub.com/Azure/ARO-HCP/test/e2e.init.func19.1.1({0x66?, 0x0?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:115 +0xa8 fp=0xc000e60e88 sp=0xc000e60c40 pc=0x5b49488\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x7fd5e40ff460?, 0xc00089faa0?}, 0x0?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.34.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc000e60ee0 sp=0xc000e60e88 pc=0x3581842\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x7fd5e40ff460, 0xc00089faa0}, {0x12a05f200, 0x4000000000000000, 0x3fb999999999999a, 0x19, 0xa7a358200}, 0xc000e61540)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc000e60f58 sp=0xc000e60ee0 pc=0x357f8ee\ngithub.com/Azure/ARO-HCP/test/e2e.init.func19.1({0x7fd5e40ff460, 0xc00089faa0}, {0x81bbd5c, 0x4}, {0x81c3d5c, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:108 +0xb38 fp=0xc000e617f0 sp=0xc000e60f58 pc=0x5b48f78\nruntime.call64(0xc00089fb60, 0x85a90b8, 0x0, 0x0, 0x0, 0x30, 0xc000e61a78)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:776 +0x43 fp=0xc000e61840 sp=0xc000e617f0 pc=0x2dbd363\nruntime.reflectcall(0x70bc6c0?, 0x8be7210?, 0x8?, 0x81fbd86?, 0x0?, 0x12?, 0x70bc6c0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000e61880 sp=0xc000e61840 pc=0x2dc5ab6\nreflect.Value.call({0x7309280?, 0x85a90b8?, 0x13?}, {0x81bbe44, 0x4}, {0xc000a1b2c0, 0x3, 0x3?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000e61de0 sp=0xc000e61880 pc=0x2df8a86\nreflect.Value.Call({0x7309280?, 0x85a90b8?, 0x8d4f0b0?}, {0xc000a1b2c0?, 0x30?, 0x730f260?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000e61e58 sp=0xc000e61de0 pc=0x2df7b79\ngithub.com/onsi/ginkgo/v2.invokeFunction({0x7309280, 0x85a90b8}, {0xc00089fb30, 0x3, 0x730f260?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5 fp=0xc000e61f18 sp=0xc000e61e58 pc=0x323c2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0x8cc8d40?, 0xc00089faa0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc000e61f88 sp=0xc000e61f18 pc=0x323bfc5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000e61fe0 sp=0xc000e61f88 pc=0x322dabb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e61fe8 sp=0xc000e61fe0 pc=0x2dbecc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 117\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax 0xca\nrbx 0x0\nrcx 0x2dc0ae3\nrdx 0x0\nrdi 0xc1ba5b0\nrsi 0x80\nrbp 0x7fff607392a0\nrsp 0x7fff60739258\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0xc0000de008\nr13 0x1\nr14 0xc1b84c0\nr15 0x3\nrip 0x2dc0ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command\namd64.s:1693 +0x1 fp=0xc000bf1fe8 sp=0xc000bf1fe0 pc=0x2dbecc1\ncreated by github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests.ExtensionTestSpecs.Run in goroutine 1\n\t/go/pkg/mod/github.com/openshift-eng/openshift-tests-extension@v0.0.0-20260127124016-0fed2b824818/pkg/extension/extensiontests/spec.go:230 +0x1e5\n\ngoroutine 119 gp=0xc000bb48c0 m=nil [select]:\nruntime.gopark(0xc000e602f0?, 0x2?, 0xa8?, 0xf8?, 0xc000e602e4?)\n\t/usr/lib/golang/src/runtime/proc.go:460 +0xce fp=0xc000e60170 sp=0xc000e60150 pc=0x2db650e\nruntime.selectgo(0xc000e602f0, 0xc000e602e0, 0xc000b8d940?, 0x0, 0x35cdd3c?, 0x1)\n\t/usr/lib/golang/src/runtime/select.go:351 +0x8b7 fp=0xc000e602b0 sp=0xc000e60170 pc=0x2d912f7\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/internal/shared.Delay({0x8cb0988, 0xc0009b4460}, 0x81dda58?)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/internal/shared/shared.go:36 +0x76 fp=0xc000e60320 sp=0xc000e602b0 pc=0x35cd496\ngithub.com/Azure/azure-sdk-for-go/sdk/azcore/runtime.(*Poller[...]).PollUntilDone(0x8cebf60, {0x8cb0988, 0xc0009b4460}, 0x0)\n\t/go/pkg/mod/github.com/!azure/azure-sdk-for-go/sdk/azcore@v1.21.0/runtime/poller.go:284 +0x7b2 fp=0xc000e60668 sp=0xc000e60320 pc=0x3abad52\ngithub.com/Azure/ARO-HCP/test/util/framework.CreateHCPClusterAndWait({0x8cb09f8?, 0xc000340fc0?}, {{0x8cc8c68?, 0xc000638b80?}, 0xc00120a680?}, 0xc000e60b70, {0xc000ea4260, 0x1e}, {0xc000bc2440, 0x1b}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/hcp_helper.go:694 +0x405 fp=0xc000e608e0 sp=0xc000e60668 pc=0x5261e85\ngithub.com/Azure/ARO-HCP/test/util/framework.(*perItOrDescribeTestContext).CreateHCPClusterFromParam(_, {_, _}, {{_, _}, _}, {_, _}, {{0xc00127d658, 0x7}, ...}, ...)\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/util/framework/deployment_helper.go:346 +0x497 fp=0xc000e60c40 sp=0xc000e608e0 pc=0x5255fd7\ngithub.com/Azure/ARO-HCP/test/e2e.init.func19.1.1({0x66?, 0x0?})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:115 +0xa8 fp=0xc000e60e88 sp=0xc000e60c40 pc=0x5b49488\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtectionWithContext({0x7fd5e40ff460?, 0xc00089faa0?}, 0x0?)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.34.3/pkg/util/wait/wait.go:159 +0x62 fp=0xc000e60ee0 sp=0xc000e60e88 pc=0x3581842\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoffWithContext({0x7fd5e40ff460, 0xc00089faa0}, {0x12a05f200, 0x4000000000000000, 0x3fb999999999999a, 0x19, 0xa7a358200}, 0xc000e61540)\n\t/go/pkg/mod/k8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:501 +0xae fp=0xc000e60f58 sp=0xc000e60ee0 pc=0x357f8ee\ngithub.com/Azure/ARO-HCP/test/e2e.init.func19.1({0x7fd5e40ff460, 0xc00089faa0}, {0x81bbd5c, 0x4}, {0x81c3d5c, 0x7})\n\t/opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:108 +0xb38 fp=0xc000e617f0 sp=0xc000e60f58 pc=0x5b48f78\nruntime.call64(0xc00089fb60, 0x85a90b8, 0x0, 0x0, 0x0, 0x30, 0xc000e61a78)\n\t/usr/lib/golang/src/runtime/asm_amd64.s:776 +0x43 fp=0xc000e61840 sp=0xc000e617f0 pc=0x2dbd363\nruntime.reflectcall(0x70bc6c0?, 0x8be7210?, 0x8?, 0x81fbd86?, 0x0?, 0x12?, 0x70bc6c0?)\n\t\u003cautogenerated\u003e:1 +0x36 fp=0xc000e61880 sp=0xc000e61840 pc=0x2dc5ab6\nreflect.Value.call({0x7309280?, 0x85a90b8?, 0x13?}, {0x81bbe44, 0x4}, {0xc000a1b2c0, 0x3, 0x3?})\n\t/usr/lib/golang/src/reflect/value.go:581 +0xcc6 fp=0xc000e61de0 sp=0xc000e61880 pc=0x2df8a86\nreflect.Value.Call({0x7309280?, 0x85a90b8?, 0x8d4f0b0?}, {0xc000a1b2c0?, 0x30?, 0x730f260?})\n\t/usr/lib/golang/src/reflect/value.go:365 +0xb9 fp=0xc000e61e58 sp=0xc000e61de0 pc=0x2df7b79\ngithub.com/onsi/ginkgo/v2.invokeFunction({0x7309280, 0x85a90b8}, {0xc00089fb30, 0x3, 0x730f260?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:339 +0x2c5 fp=0xc000e61f18 sp=0xc000e61e58 pc=0x323c2c5\ngithub.com/onsi/ginkgo/v2.generateTable.func2.1({0x8cc8d40?, 0xc00089faa0?})\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/table_dsl.go:293 +0x105 fp=0xc000e61f88 sp=0xc000e61f18 pc=0x323bfc5\ngithub.com/onsi/ginkgo/v2/internal.(*Suite).runNode.func3()\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:901 +0x7b fp=0xc000e61fe0 sp=0xc000e61f88 pc=0x322dabb\nruntime.goexit({})\n\t/usr/lib/golang/src/runtime/asm_amd64.s:1693 +0x1 fp=0xc000e61fe8 sp=0xc000e61fe0 pc=0x2dbecc1\ncreated by github.com/onsi/ginkgo/v2/internal.(*Suite).runNode in goroutine 117\n\t/go/pkg/mod/github.com/openshift/onsi-ginkgo/v2@v2.6.1-0.20250416174521-4eb003743b54/internal/suite.go:888 +0xd50\n\nrax 0xca\nrbx 0x0\nrcx 0x2dc0ae3\nrdx 0x0\nrdi 0xc1ba5b0\nrsi 0x80\nrbp 0x7fff607392a0\nrsp 0x7fff60739258\nr8 0x0\nr9 0x0\nr10 0x0\nr11 0x286\nr12 0xc0000de008\nr13 0x1\nr14 0xc1b84c0\nr15 0x3\nrip 0x2dc0ae1\nrflags 0x286\ncs 0x33\nfs 0x0\ngs 0x0\nCommand Error: exit status 2\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"53c6fe243d9d7ed54d5c3e1c929e4e00c871c3dbf1500a51cf0525362d6bd6bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"4ff69f402a35f8589728a46de86bc483f764602ca78dd11c5bdc70a78155a06e","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001214ba0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-jb2xfp, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jb2xfp\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-jb2xfp, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-jb2xfp\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00058d540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-jb2xfp\",\n },\n \u003c*fmt.wrapError | 0xc0008b85e0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-jb2xfp\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:124]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001214ba0\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-jb2xfp, error: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-jb2xfp\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-jb2xfp, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-jb2xfp\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00058d540\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group autoscaling-cluster-jb2xfp\",\n },\n \u003c*fmt.wrapError | 0xc0008b85e0\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"autoscaling-hcp-cluster\\\" in resourcegroup=\\\"autoscaling-cluster-jb2xfp\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded"}},{"row_id":"58d83a10ee5aca75295ca907ffa9b455328811c665954f6da0644c31b3a7ee88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"1afd98ace371047c0b6f30472486f90f36627888de8ef560568f39a8a9f7eefe","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00152e090\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-w2k2l9\" in resourcegroup=\"admin-credential-lifecycle-test-xnk8kd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-w2k2l9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xnk8kd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dd89c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00152e090\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-w2k2l9\" in resourcegroup=\"admin-credential-lifecycle-test-xnk8kd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-w2k2l9\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-xnk8kd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000dd89c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-w2k2l9 in resource group admin-credential-lifecycle-test-xnk8kd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"6442c7a0a53c88ba60605936587229e151fe1a57e4358d914a905523f7028dea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"db21d7b4d2180dfb67585d94a9c18b9307efdc3021536e3fe62d1bfcb1343968","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e62d50\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-rrjgvj\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rrjgvj, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-rrjgvj\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rrjgvj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00047a540\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rrjgvj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e62d50\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-rrjgvj\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rrjgvj, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-rrjgvj\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rrjgvj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00047a540\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-rrjgvj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"72ec6e08151598821a2de5d60f0d2cc9fbd164d497161b3acef9dcbe5737f824","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"a7a3a791a16cdc30955425371d11ec180f558068ee2777f1b760ec0eec8c71db","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526605 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/07/26 01:50:05.987\n \"ts\"=\"2026-04-07 01:50:05.987526\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-cgm98g-zfqtjt\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:50:10.608\n STEP: creating customer resources @ 04/07/26 01:50:10.697\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 01:51:30.392\n \"ts\"=\"2026-04-07 01:51:30.392820\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-cgm98g\" \"resourceGroup\"=\"rg-np-version-upgrade-cgm98g-zfqtjt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526605 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/07/26 01:50:05.987\n \"ts\"=\"2026-04-07 01:50:05.987526\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-cgm98g-zfqtjt\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:50:10.608\n STEP: creating customer resources @ 04/07/26 01:50:10.697\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 01:51:30.392\n \"ts\"=\"2026-04-07 01:51:30.392820\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-cgm98g\" \"resourceGroup\"=\"rg-np-version-upgrade-cgm98g-zfqtjt\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"791c4d0e7e57dcc614e10a293c0503df8ef685ee38fe4ab105d3a589f1619d4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"6d1fba2228413ee88801c71b444cc9f8d5620d8f429f71bb349c575fb7b5f3b0","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00158ce10\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-dqsrr2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-dqsrr2, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dqsrr2\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-dqsrr2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7e9c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-dqsrr2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00158ce10\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-dqsrr2\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-dqsrr2, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-dqsrr2\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-dqsrr2, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7e9c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-dqsrr2\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"820e4919dc95dfa15030929cc28647f59eb936deff5ca1722e7260a57b5131e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"ea06de1114c3d8f9ae2bd132acedc5c7fdc738b85c2b59230a4f78070b8f7429","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526656 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 01:50:57.267\n \"ts\"=\"2026-04-07 01:50:57.267206\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-mn8s95-8qq489\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:51:01.017\n STEP: creating customer resources @ 04/07/26 01:51:01.099\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 01:52:22.852\n \"ts\"=\"2026-04-07 01:52:22.852166\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-mn8s95\" \"resourceGroup\"=\"rg-np-version-upgrade-mn8s95-8qq489\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775526656 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 01:50:57.267\n \"ts\"=\"2026-04-07 01:50:57.267206\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-mn8s95-8qq489\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:51:01.017\n STEP: creating customer resources @ 04/07/26 01:51:01.099\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 01:52:22.852\n \"ts\"=\"2026-04-07 01:52:22.852166\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-mn8s95\" \"resourceGroup\"=\"rg-np-version-upgrade-mn8s95-8qq489\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"a7ba40847624d4932d537b4985f9cb7b773f7a60ff17b774074b3e8916f83494","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"7ed989922bc03f5a77baa84831fdc451a05ac58926599595e31930fd3a9d61de","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000930930\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-2mpbcw, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2mpbcw\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-2mpbcw, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2mpbcw\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000995180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-2mpbcw\",\n },\n \u003c*fmt.wrapError | 0xc001406c00\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2mpbcw\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:145]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000930930\u003e: \n failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-2mpbcw, error: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-2mpbcw\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-2mpbcw, error: failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2mpbcw\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000995180\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool np-1 in resource group pullsecret-test-2mpbcw\",\n },\n \u003c*fmt.wrapError | 0xc001406c00\u003e{\n msg: \"failed waiting for nodepool=\\\"np-1\\\" for cluster \\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-2mpbcw\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded"}},{"row_id":"bc5e7716d9437eb6c5fcf251aeb15c4690a016baf94ecf3d484ddb2c890e8397","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"bcebc3a2c05425e0449d8d141ada09157b91cc2430df5a0d55489035341137ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"5e6c3c5c47b005967ec25cd98cba3a5126b257a4788d48b77e5a129547e95eac","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010923c0\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-rn4rfw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-rn4rfw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-rn4rfw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-rn4rfw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a21460\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-rn4rfw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010923c0\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-rn4rfw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-rn4rfw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-rn4rfw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-rn4rfw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a21460\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-rn4rfw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"ceb3fe9d27fd42d1d6b2f5008767afb7fe35f71c63bf2481054b6f7a9534fe0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"bc0af7ca1eb571869bcfd45d0933b754212864c3ca865d87bc33745cfdc01aa5","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00152a210\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5wwxjp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-5wwxjp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00098cbd0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp\",\n },\n \u003c*fmt.wrapErrors | 0xc00152a1e0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-5wwxjp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00098cbd0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:93]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00152a210\u003e: \n failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5wwxjp\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster ea-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-5wwxjp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00098cbd0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp\",\n },\n \u003c*fmt.wrapErrors | 0xc00152a1e0\u003e{\n msg: \"failed waiting for cluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-5wwxjp\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00098cbd0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster ea-cluster in resource group external-auth-cluster-5wwxjp\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"d8c0220988c49bd3c44b27e121f9c8e9e3780ddacc7182b9587bc4a9517677da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"513321780d22667c85d39965dcd485e1739ec694dc7dfb6d2311d111c6478061","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009a8750\u003e: \n failed to create HCP cluster cluster-zstream-4-21-fdsksh, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: failed waiting for cluster=\"cluster-zstream-4-21-fdsksh\" in resourcegroup=\"rg-zstream-upgrade-4-21-ssph6j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-fdsksh, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: failed waiting for cluster=\\\"cluster-zstream-4-21-fdsksh\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-ssph6j\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012d10b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j\",\n },\n \u003c*fmt.wrapErrors | 0xc0009a8720\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-fdsksh\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-ssph6j\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012d10b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009a8750\u003e: \n failed to create HCP cluster cluster-zstream-4-21-fdsksh, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: failed waiting for cluster=\"cluster-zstream-4-21-fdsksh\" in resourcegroup=\"rg-zstream-upgrade-4-21-ssph6j\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-fdsksh, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: failed waiting for cluster=\\\"cluster-zstream-4-21-fdsksh\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-ssph6j\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012d10b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j\",\n },\n \u003c*fmt.wrapErrors | 0xc0009a8720\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-fdsksh\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-ssph6j\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0012d10b0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-21-fdsksh in resource group rg-zstream-upgrade-4-21-ssph6j\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"da3800143efa67c5fb397594cf86c1d1f595ca699614ee60525eb7be3a6cd95e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"84b0d7cc081dfd6750b672d63540a59e19f9f30758d835bf5f5ef823dbac3aa5","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775525311 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/07/26 01:28:31.216\n \"ts\"=\"2026-04-07 01:28:31.216502\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n STEP: creating cluster parameters @ 04/07/26 01:28:36.315\n STEP: creating customer resources @ 04/07/26 01:28:36.315\n STEP: creating the HCP cluster @ 04/07/26 01:29:58.448\n \"ts\"=\"2026-04-07 01:29:58.449091\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91 @ 04/07/26 02:14:58.449\n \"ts\"=\"2026-04-07 02:14:58.449977\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 02:14:58.450050\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-7c8wc2\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n \"ts\"=\"2026-04-07 02:14:58.733016\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-7c8wc2\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 02:15:02.047844\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-07 02:15:02.048091\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-07 02:15:02.048186\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 02:15:02.049536\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\"\n \"ts\"=\"2026-04-07 02:15:02.049561\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n \"ts\"=\"2026-04-07 02:15:02.217441\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 02:15:02.356982\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\" \"deployment\"=\"msi-deployment-16\"\n \"ts\"=\"2026-04-07 02:15:02.360274\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-07 02:15:02.682590\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-7c8wc2\"\n \"ts\"=\"2026-04-07 02:15:03.008994\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 02:15:03.009116\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 02:15:03.009272\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n \"ts\"=\"2026-04-07 02:24:38.523395\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n \"ts\"=\"2026-04-07 02:24:38.523422\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775525311 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/07/26 01:28:31.216\n \"ts\"=\"2026-04-07 01:28:31.216502\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n STEP: creating cluster parameters @ 04/07/26 01:28:36.315\n STEP: creating customer resources @ 04/07/26 01:28:36.315\n STEP: creating the HCP cluster @ 04/07/26 01:29:58.448\n \"ts\"=\"2026-04-07 01:29:58.449091\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:91 @ 04/07/26 02:14:58.449\n \"ts\"=\"2026-04-07 02:14:58.449977\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 02:14:58.450050\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-7c8wc2\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n \"ts\"=\"2026-04-07 02:14:58.733016\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-7c8wc2\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 02:15:02.047844\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-07 02:15:02.048091\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-07 02:15:02.048186\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 02:15:02.049536\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\"\n \"ts\"=\"2026-04-07 02:15:02.049561\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n \"ts\"=\"2026-04-07 02:15:02.217441\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 02:15:02.356982\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-16\" \"deployment\"=\"msi-deployment-16\"\n \"ts\"=\"2026-04-07 02:15:02.360274\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-07 02:15:02.682590\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-7c8wc2\"\n \"ts\"=\"2026-04-07 02:15:03.008994\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 02:15:03.009116\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 02:15:03.009272\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n \"ts\"=\"2026-04-07 02:24:38.523395\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\n \"ts\"=\"2026-04-07 02:24:38.523422\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-labels-taints-j4gv7q\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e20b433b9a52f9d9de2f9d083d0ea285c3258dc236b340dcf54ab163c3a74332","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"38bb290cdcd5fb0fd6fc57972bfe6e55acd2c90b588687aa9b866e4af14f4e0f","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001604390\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nszsx5\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-nszsx5\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009757d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5\",\n },\n \u003c*fmt.wrapError | 0xc000e161a0\u003e{\n msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-nszsx5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001604390\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-nszsx5\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5, error: failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-nszsx5\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009757d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-nszsx5\",\n },\n \u003c*fmt.wrapError | 0xc000e161a0\u003e{\n msg: \"failed waiting for nodepool=\\\"nodepool-128GiB\\\" for cluster \\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-nszsx5\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded"}},{"row_id":"e523a182f4a8abc306e31ab56450d3cdc5648bb027309402a388d8530f523300","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"45549f3d568ceae199450b06a0f29ca707fdcd572e216b08b615100bc902a118","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008d4210\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jwszks\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jwszks\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e02710\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks\",\n },\n \u003c*fmt.wrapErrors | 0xc0008d41e0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jwszks\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e02710\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008d4210\u003e: \n failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-jwszks\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster node-limit-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jwszks\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e02710\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks\",\n },\n \u003c*fmt.wrapErrors | 0xc0008d41e0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-jwszks\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e02710\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster node-limit-cluster in resource group node-limit-test-jwszks\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-gsft5q\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-gsft5q, error: context deadline exceeded"}},{"row_id":"ec76dc3b2c560de732e4dbcf962f7e867a7a709c163f40df87cdc69131b2c34a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:184]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:184]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":30,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":32,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T00:13:02Z"},"failed_at":["e2e","unknown"],"failed_test_count":11,"occurrences":[{"row_id":"13e481060039a74baf2427b3d0834f73fc26b8ee9e3b3d9fdff9885a5831f601","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"203e2c7ffff61cb7e8a9241ca3de6134343765d2bd071522facb4429605fa8c3","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00181f6e0\u003e: \n failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-jb4xcc\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-jb4xcc, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-jb4xcc\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-jb4xcc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008dc420\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-jb4xcc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00181f6e0\u003e: \n failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-jb4xcc\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-jb4xcc, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-jb4xcc\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-jb4xcc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0008dc420\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster hcp-cluster-np-128 in resource group clusternp128-jb4xcc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"16bbf46170d03ba4e56379ecedb39fcc2e17eed44515ff426b3f56aeef3f1380","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"5be661969e52e9a052ab0134a9f3bdb6e4e82b3b6b5d2c04656940f3faa90b53","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001558fc0\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-5cmtfg\" in resourcegroup=\"rg-zstream-upgrade-4-19-tsqtqw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-5cmtfg in resource group rg-zstream-upgrade-4-19-tsqtqw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-5cmtfg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-tsqtqw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-5cmtfg in resource group rg-zstream-upgrade-4-19-tsqtqw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0007fdb60\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-5cmtfg in resource group rg-zstream-upgrade-4-19-tsqtqw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001558fc0\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-5cmtfg\" in resourcegroup=\"rg-zstream-upgrade-4-19-tsqtqw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-5cmtfg in resource group rg-zstream-upgrade-4-19-tsqtqw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-5cmtfg\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-tsqtqw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-5cmtfg in resource group rg-zstream-upgrade-4-19-tsqtqw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0007fdb60\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-5cmtfg in resource group rg-zstream-upgrade-4-19-tsqtqw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"2ade379fc3cc8f8d45a63281798341fbe6fe631f4d153a81998cdbddfdde000d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"8a59fc555efc0c6395952c408b943ce13eb2659f4641992e1435b4f7478938ce","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0016a1620\u003e: \n failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-m987vc\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-m987vc, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-m987vc\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-m987vc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000765c80\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-m987vc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:155]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0016a1620\u003e: \n failed waiting for hcpCluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-m987vc\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-m987vc, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"pullsecret-hcp-cluster\\\" in resourcegroup=\\\"pullsecret-test-m987vc\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-m987vc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000765c80\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster pullsecret-hcp-cluster in resource group pullsecret-test-m987vc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"3080d267b9ddb1f993d6a7e22e0cc01aba14014a215b0f7a2dd37fd496860204","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"b7d39ccaa33471c6fe6d2423539073913dae6879ccbe030526e8f6b3ab342f0b","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775524781 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 01:19:48.537\n \"ts\"=\"2026-04-07 01:19:48.537377\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-hw8d7b-thhkdf\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:19:52.971\n STEP: creating customer resources @ 04/07/26 01:19:53.063\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 01:21:09.322\n \"ts\"=\"2026-04-07 01:21:09.322669\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-hw8d7b\" \"resourceGroup\"=\"rg-np-version-upgrade-hw8d7b-thhkdf\"\n STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 01:33:26.698\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775524781 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 01:19:48.537\n \"ts\"=\"2026-04-07 01:19:48.537377\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-hw8d7b-thhkdf\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:19:52.971\n STEP: creating customer resources @ 04/07/26 01:19:53.063\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 01:21:09.322\n \"ts\"=\"2026-04-07 01:21:09.322669\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-hw8d7b\" \"resourceGroup\"=\"rg-np-version-upgrade-hw8d7b-thhkdf\"\n STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 01:33:26.698\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"393c0bd83f84821fadad6e080bd16e866ccb32f9e029ffba7285c5a07e90be6d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"23cb7cad2d50cb707def47ef79150e901e0494d1b2b7151fbbaefbb3e6bea082","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015307b0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-grxshz\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-vdfx67\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a8c4b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015307b0\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-grxshz\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-vdfx67\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a8c4b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"5af455725758ae1b841f50594afdc54ea5baad13e18f4062cba851fec21387bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"611671fe68b945ff638638addf68f7bf0d9efbfc195d1ac82a6d593af263e1d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"697cb4e2d08f75c3268c0a07e907dadf1330b74f8378572502578afac4409d4c","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001365830\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-p9jg98\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-p9jg98, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-p9jg98\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-p9jg98, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000aeda50\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-p9jg98\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001365830\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-p9jg98\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-p9jg98, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-p9jg98\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-p9jg98, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000aeda50\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-p9jg98\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"8d9f11f469783827fcd515b928b8462ccbbc7dbe0286bad61324ff86cf51bc46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"becaebe81abc8fbb81aaadb376c5d1d5b254498b9ea945ec6a9e215bb1ffaa52","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dcc390\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-sv4k2p\" in resourcegroup=\"rg-stable-4-20-sjfcpf\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-sv4k2p in resource group rg-stable-4-20-sjfcpf, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-sv4k2p\\\" in resourcegroup=\\\"rg-stable-4-20-sjfcpf\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-sv4k2p in resource group rg-stable-4-20-sjfcpf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a3e850\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-sv4k2p in resource group rg-stable-4-20-sjfcpf\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dcc390\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-sv4k2p\" in resourcegroup=\"rg-stable-4-20-sjfcpf\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-sv4k2p in resource group rg-stable-4-20-sjfcpf, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-sv4k2p\\\" in resourcegroup=\\\"rg-stable-4-20-sjfcpf\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-sv4k2p in resource group rg-stable-4-20-sjfcpf, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a3e850\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-sv4k2p in resource group rg-stable-4-20-sjfcpf\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}},{"row_id":"c9a5c10aef613252a2bbc584915cc89970b51611d0b083e9bd0d2e3bd1c89c2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-07T02:16:09Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"d8f67ed8be774c0ca967fd731fd340f25cce836dd0f2e251546b731702960f66","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"fe9959356e6bebc75442b36c9b6e15a39d8864f5ee94b77970582f70eb6485ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","failed_at":"e2e","signature_id":"fce7f9cb30c215532540321a2400b35a1f573a9281d5e92cf3123f1c7f7dacda","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013c2210\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-fqshxl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-fqshxl, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-fqshxl\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-fqshxl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a96ca0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-fqshxl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013c2210\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-fqshxl\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-fqshxl, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-fqshxl\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-fqshxl, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a96ca0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-fqshxl\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-grxshz\" in resourcegroup=\"admin-credential-lifecycle-test-vdfx67\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-grxshz in resource group admin-credential-lifecycle-test-vdfx67, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308037523705856","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-07T00:12:16Z"},"failed_at":["e2e","unknown"],"failed_test_count":2,"occurrences":[{"row_id":"66e0f67608cc9c4225181656b1fd227921519b60832781556d73d4f021e75ae9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308037523705856","occurred_at":"2026-04-07T00:12:16Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-07T02:14:40Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"dcbc08230126683480a18ddfc3749b6f168f4daf958e936556eed48030fbbee0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308037523705856","occurred_at":"2026-04-07T00:12:16Z","failed_at":"e2e","signature_id":"ab4eb11ba454086e3a61bca639145624e40a3a0c90bd62c730548e8ec4d56dc9","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775524025 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 01:07:05.535\n \"ts\"=\"2026-04-07 01:07:05.535184\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:07:08.385\n STEP: creating customer resources @ 04/07/26 01:07:08.466\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 01:08:01.242\n \"ts\"=\"2026-04-07 01:08:01.242928\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gg5w6p\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\n STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 01:21:28.227\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154 @ 04/07/26 02:06:28.229\n \"ts\"=\"2026-04-07 02:06:28.229894\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 02:06:28.229987\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gg5w6p-86c4xv\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\n \"ts\"=\"2026-04-07 02:06:28.331005\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-gg5w6p-86c4xv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 02:06:28.637569\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-99031ad0c929f4b469b552900cdb42a4d2445ad8090c3f5e30a3cba67f4c8788.yaml.gz\"\n \"ts\"=\"2026-04-07 02:06:28.637687\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-99031ad0c929f4b469b552900cdb42a4d2445ad8090c3f5e30a3cba67f4c8788.yaml.gz\"\n \"ts\"=\"2026-04-07 02:06:28.637757\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 02:06:28.639289\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-103\"\n \"ts\"=\"2026-04-07 02:06:28.639301\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\n \"ts\"=\"2026-04-07 02:06:28.711446\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\" \"deployment\"=\"pooledNonMsiScopedAssignments-6bqq3eelft2oc\"\n \"ts\"=\"2026-04-07 02:06:28.790571\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-103\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 02:06:28.870725\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gg5w6p-86c4xv\"\n \"ts\"=\"2026-04-07 02:06:28.939396\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-103\" \"deployment\"=\"msi-deployment-103\"\n \"ts\"=\"2026-04-07 02:06:29.124477\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 02:06:29.124591\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 02:06:29.124716\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775524025 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:234\n STEP: creating resource group @ 04/07/26 01:07:05.535\n \"ts\"=\"2026-04-07 01:07:05.535184\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\n STEP: creating cluster parameters at control plane version @ 04/07/26 01:07:08.385\n STEP: creating customer resources @ 04/07/26 01:07:08.466\n STEP: creating the HCP cluster with version 4.21.7 @ 04/07/26 01:08:01.242\n \"ts\"=\"2026-04-07 01:08:01.242928\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gg5w6p\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\n STEP: creating nodepool with version 4.21.5 (behind control plane; upgrade path validated via Cincinnati) @ 04/07/26 01:21:28.227\n [FAILED] in [It] - /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154 @ 04/07/26 02:06:28.229\n \"ts\"=\"2026-04-07 02:06:28.229894\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-07 02:06:28.229987\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gg5w6p-86c4xv\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\n \"ts\"=\"2026-04-07 02:06:28.331005\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-gg5w6p-86c4xv\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-07 02:06:28.637569\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-99031ad0c929f4b469b552900cdb42a4d2445ad8090c3f5e30a3cba67f4c8788.yaml.gz\"\n \"ts\"=\"2026-04-07 02:06:28.637687\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-99031ad0c929f4b469b552900cdb42a4d2445ad8090c3f5e30a3cba67f4c8788.yaml.gz\"\n \"ts\"=\"2026-04-07 02:06:28.637757\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-07 02:06:28.639289\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-103\"\n \"ts\"=\"2026-04-07 02:06:28.639301\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\n \"ts\"=\"2026-04-07 02:06:28.711446\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\" \"deployment\"=\"pooledNonMsiScopedAssignments-6bqq3eelft2oc\"\n \"ts\"=\"2026-04-07 02:06:28.790571\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-103\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-07 02:06:28.870725\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gg5w6p-86c4xv\"\n \"ts\"=\"2026-04-07 02:06:28.939396\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-103\" \"deployment\"=\"msi-deployment-103\"\n \"ts\"=\"2026-04-07 02:06:29.124477\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-07 02:06:29.124591\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-07 02:06:29.124716\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gg5w6p-r2dqkz\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}