{"meta":{"date":"2026-04-03","timezone":"UTC","generated_at":"2026-08-30T11:02:35Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":5,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-03T18:45:33Z"},"failed_at":["e2e","unknown"],"failed_test_count":14,"occurrences":[{"row_id":"1aae57e4d6cc8a175d38ab6a53a79ee96fcc0f32578cdcb6593143169413e372","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"a8d31a96b11e8a03efdf422a533fe81c28e318cf4353c55b406a4488dbb92f45","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775245707 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\n STEP: creating resource group @ 04/03/26 19:48:28.242\n \"ts\"=\"2026-04-03 19:48:28.242687\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 19:48:32.172\n STEP: creating customer resources @ 04/03/26 19:48:32.273\n STEP: creating the HCP cluster with version 4.20.17 @ 04/03/26 19:49:34.027\n \"ts\"=\"2026-04-03 19:49:34.027707\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gh68jv\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 20:01:59.339\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 20:11:15.755\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 20:13:15.571\n STEP: capturing node release images before upgrade @ 04/03/26 20:13:15.748\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/03/26 20:13:15.951\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 20:14:02.797\n STEP: verifying node pool GET still reflects the new version @ 04/03/26 20:42:11.233\n \"ts\"=\"2026-04-03 20:42:12.161810\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 20:42:12.161939\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gh68jv-rmkr2d\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n \"ts\"=\"2026-04-03 20:42:12.456525\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-gh68jv-rmkr2d\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 20:42:14.379525\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-03 20:42:14.379643\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-03 20:42:14.379720\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 20:42:14.381290\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\"\n \"ts\"=\"2026-04-03 20:42:14.381308\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n \"ts\"=\"2026-04-03 20:42:14.597243\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\" \"deployment\"=\"pooledNonMsiScopedAssignments-gyhigscniinga\"\n \"ts\"=\"2026-04-03 20:42:14.870794\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 20:42:14.912330\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gh68jv-rmkr2d\"\n \"ts\"=\"2026-04-03 20:42:15.466293\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\" \"deployment\"=\"msi-deployment-45\"\n \"ts\"=\"2026-04-03 20:42:15.973131\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 20:42:15.973254\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 20:42:15.973386\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775245707 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\n STEP: creating resource group @ 04/03/26 19:48:28.242\n \"ts\"=\"2026-04-03 19:48:28.242687\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 19:48:32.172\n STEP: creating customer resources @ 04/03/26 19:48:32.273\n STEP: creating the HCP cluster with version 4.20.17 @ 04/03/26 19:49:34.027\n \"ts\"=\"2026-04-03 19:49:34.027707\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gh68jv\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 20:01:59.339\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 20:11:15.755\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 20:13:15.571\n STEP: capturing node release images before upgrade @ 04/03/26 20:13:15.748\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/03/26 20:13:15.951\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 20:14:02.797\n STEP: verifying node pool GET still reflects the new version @ 04/03/26 20:42:11.233\n \"ts\"=\"2026-04-03 20:42:12.161810\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 20:42:12.161939\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gh68jv-rmkr2d\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n \"ts\"=\"2026-04-03 20:42:12.456525\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-gh68jv-rmkr2d\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 20:42:14.379525\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-03 20:42:14.379643\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-03 20:42:14.379720\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 20:42:14.381290\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\"\n \"ts\"=\"2026-04-03 20:42:14.381308\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n \"ts\"=\"2026-04-03 20:42:14.597243\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\" \"deployment\"=\"pooledNonMsiScopedAssignments-gyhigscniinga\"\n \"ts\"=\"2026-04-03 20:42:14.870794\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 20:42:14.912330\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gh68jv-rmkr2d\"\n \"ts\"=\"2026-04-03 20:42:15.466293\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\" \"deployment\"=\"msi-deployment-45\"\n \"ts\"=\"2026-04-03 20:42:15.973131\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 20:42:15.973254\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 20:42:15.973386\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"2c87d6b0559cdcee3aa98d53263486f423b97c501f08173ce921c9f511027947","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"4e2f5684d9d2905047d799108ef5b884b830a3db0900c3cc4ffe6a45b4bcc121","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"520eafa06c89c4752bee7811e891867fb3257bfcb33fdc483d9269e770f97de2","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"\"=\"0\" \"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-fsz9k\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-03 20:34:41.587768\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-hzcv6\" \"labels\"={\"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"topology.kubernetes.io/zone\"=\"1\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"key1\"=\"value1\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-hzcv6\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/managed\"=\"true\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\" \"beta.kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-03 20:34:41.587818\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-k6pvw\" \"labels\"={\"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-k6pvw\" \"node-role.kubernetes.io/worker\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"key1\"=\"value1\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n STEP: verifying original labels persist on 3 nodes @ 04/03/26 20:34:41.587\n STEP: verifying original taints persist on 2 nodes @ 04/03/26 20:34:41.587\n \"ts\"=\"2026-04-03 20:34:41.588122\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 20:34:41.588264\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-44fz9p\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\n \"ts\"=\"2026-04-03 20:34:41.767633\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-44fz9p\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 20:34:43.091201\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-03 20:34:43.091299\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-03 20:34:43.091358\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 20:34:43.092891\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\"\n \"ts\"=\"2026-04-03 20:34:43.092910\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\n \"ts\"=\"2026-04-03 20:34:43.282140\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-03 20:34:43.445177\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 20:34:43.456328\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-44fz9p\"\n \"ts\"=\"2026-04-03 20:34:43.747143\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\" \"deployment\"=\"msi-deployment-96\"\n \"ts\"=\"2026-04-03 20:34:44.098708\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 20:34:44.098863\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 20:34:44.098996\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\"=\"0\" \"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-fsz9k\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-03 20:34:41.587768\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-hzcv6\" \"labels\"={\"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"topology.kubernetes.io/zone\"=\"1\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"key1\"=\"value1\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-hzcv6\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/managed\"=\"true\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\" \"beta.kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-03 20:34:41.587818\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-k6pvw\" \"labels\"={\"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-k6pvw\" \"node-role.kubernetes.io/worker\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"key1\"=\"value1\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n STEP: verifying original labels persist on 3 nodes @ 04/03/26 20:34:41.587\n STEP: verifying original taints persist on 2 nodes @ 04/03/26 20:34:41.587\n \"ts\"=\"2026-04-03 20:34:41.588122\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 20:34:41.588264\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-44fz9p\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\n \"ts\"=\"2026-04-03 20:34:41.767633\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-44fz9p\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 20:34:43.091201\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-03 20:34:43.091299\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-03 20:34:43.091358\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 20:34:43.092891\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\"\n \"ts\"=\"2026-04-03 20:34:43.092910\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\n \"ts\"=\"2026-04-03 20:34:43.282140\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-03 20:34:43.445177\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 20:34:43.456328\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-44fz9p\"\n \"ts\"=\"2026-04-03 20:34:43.747143\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\" \"deployment\"=\"msi-deployment-96\"\n \"ts\"=\"2026-04-03 20:34:44.098708\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 20:34:44.098863\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 20:34:44.098996\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"67929d59594613ac7eb36ebc8887443966e41ce3a16ec001425cac28ada50d9f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"6fba8797e1d616ecfda25e76a09185e0e9d8e42e9f5ee6d89cfcab2fa6458a80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"74552cf58359f21f2c1dd18310fe6a3323efe248387844d4e4346d5fdbdcf78c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"f54f60621797a03b537810c47c073ed6e6a3b5a9e129109970782f85585da577","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013f0840\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-f4hz5d\" in resourcegroup=\"rg-zstream-upgrade-4-19-mf7hcn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-f4hz5d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-mf7hcn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015a34a0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013f0840\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-f4hz5d\" in resourcegroup=\"rg-zstream-upgrade-4-19-mf7hcn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-f4hz5d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-mf7hcn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0015a34a0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"b1fbc5cc6f87da4b7075cb2efc078b7cd36df9d927b2345450b83979efe5f770","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"c5cb1fe60f1abc7b4bf83fdf58154797b997a08b020af10c4a767d5de061472b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"6359997a37ab65572d65f95e29e72f2054254ddf3a4cf4ac0c8c2fc46f0a1cda","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775245500 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 19:45:00.935\n \"ts\"=\"2026-04-03 19:45:00.935972\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 19:45:03.992\n STEP: creating customer resources @ 04/03/26 19:45:04.081\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 19:46:06.079\n \"ts\"=\"2026-04-03 19:46:06.079283\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-pldnh4\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 19:59:38.878\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 20:09:14.928\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 20:11:13.489\n STEP: capturing node release images before upgrade @ 04/03/26 20:11:13.637\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 20:11:13.841\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 20:12:07.744\n STEP: verifying node pool GET still reflects the new version @ 04/03/26 20:40:16.114\n \"ts\"=\"2026-04-03 20:40:17.269118\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 20:40:17.269207\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-pldnh4-8gpfcs\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n \"ts\"=\"2026-04-03 20:40:17.673955\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-pldnh4-8gpfcs\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 20:40:19.163768\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-03 20:40:19.163906\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-03 20:40:19.163979\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 20:40:19.165684\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\"\n \"ts\"=\"2026-04-03 20:40:19.165685\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n \"ts\"=\"2026-04-03 20:40:19.357542\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\" \"deployment\"=\"pooledNonMsiScopedAssignments-7onhmt3j3ybeg\"\n \"ts\"=\"2026-04-03 20:40:19.452634\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 20:40:19.512189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-pldnh4-8gpfcs\"\n \"ts\"=\"2026-04-03 20:40:19.735871\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\" \"deployment\"=\"msi-deployment-47\"\n \"ts\"=\"2026-04-03 20:40:19.871923\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 20:40:19.872029\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 20:40:19.872148\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775245500 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 19:45:00.935\n \"ts\"=\"2026-04-03 19:45:00.935972\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 19:45:03.992\n STEP: creating customer resources @ 04/03/26 19:45:04.081\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 19:46:06.079\n \"ts\"=\"2026-04-03 19:46:06.079283\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-pldnh4\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 19:59:38.878\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 20:09:14.928\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 20:11:13.489\n STEP: capturing node release images before upgrade @ 04/03/26 20:11:13.637\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 20:11:13.841\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 20:12:07.744\n STEP: verifying node pool GET still reflects the new version @ 04/03/26 20:40:16.114\n \"ts\"=\"2026-04-03 20:40:17.269118\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 20:40:17.269207\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-pldnh4-8gpfcs\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n \"ts\"=\"2026-04-03 20:40:17.673955\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-pldnh4-8gpfcs\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 20:40:19.163768\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-03 20:40:19.163906\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-03 20:40:19.163979\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 20:40:19.165684\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\"\n \"ts\"=\"2026-04-03 20:40:19.165685\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n \"ts\"=\"2026-04-03 20:40:19.357542\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\" \"deployment\"=\"pooledNonMsiScopedAssignments-7onhmt3j3ybeg\"\n \"ts\"=\"2026-04-03 20:40:19.452634\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 20:40:19.512189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-pldnh4-8gpfcs\"\n \"ts\"=\"2026-04-03 20:40:19.735871\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\" \"deployment\"=\"msi-deployment-47\"\n \"ts\"=\"2026-04-03 20:40:19.871923\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 20:40:19.872029\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 20:40:19.872148\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"d6b22058c1beb609b5e607d14a2196be0dde53eea0b03dbbe39b70e3bfe42acc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-03T20:48:28Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}},{"row_id":"dce20f7806b5bd32d09b435ef72fe80a5671cb85b4a25d3f80b121d982aa983f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"cddfa084aa297044efcb077e7e99863ef8707f89d1167c5e6999483547025bd0","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011b0810\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2v4phv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-2v4phv\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009a5550\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0011b0810\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2v4phv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-2v4phv\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0009a5550\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"e47c8376885141906e593583d4cc2455b5785a6b454e47055fce518314fc8302","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"c225061e4e181c2491304d3cbd154d4d968c63217496bd6f9e38d71d74b4ea92","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004e0160\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w8b9cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w8b9cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015f0180\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"ea03dd36-95b8-47c7-90e1-a8dc79a919d8\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"09d15fc6-6c8e-4784-845a-952eadfe9277\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T193832Z:ea03dd36-95b8-47c7-90e1-a8dc79a919d8\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: EFBD4EB481774F60AB6EA1EDCD5ECFAF Ref B: CHI261140508025 Ref C: 2026-04-03T19:38:32Z\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"80\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/6e183501-6da1-4585-8100-e8772f21fd65\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Date\": [\n \"Fri, 03 Apr 2026 19:38:32 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0004e0000\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0004e0160\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w8b9cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w8b9cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015f0180\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"ea03dd36-95b8-47c7-90e1-a8dc79a919d8\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"09d15fc6-6c8e-4784-845a-952eadfe9277\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T193832Z:ea03dd36-95b8-47c7-90e1-a8dc79a919d8\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: EFBD4EB481774F60AB6EA1EDCD5ECFAF Ref B: CHI261140508025 Ref C: 2026-04-03T19:38:32Z\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"80\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/6e183501-6da1-4585-8100-e8772f21fd65\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Date\": [\n \"Fri, 03 Apr 2026 19:38:32 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc0004e0000\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eeb641ab29cd4a59e2b8973f21c4dd1ac07081b80826fd08573be1cbdde3a5d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"978b5249588852e82b6e0319ffa1c4ff491d36b8733adfe165bd31b6a37ca120","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eb6a20\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-l56s4b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-l56s4b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000905dd0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000eb6a20\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-l56s4b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-l56s4b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000905dd0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"f3d0f744af00afdaa1e82217ed09d137206e8a52fb0eeafd2525d10ab8f02724","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"c56aa4a27cb369c16d605912ab2f8d6cfd883cef06344a11b5dd1d06a710ba17","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ecca20\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kfr69r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kfr69r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012090e0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"c90c7b13-2d84-49c6-9e50-7c4041ceed5c\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 19:32:05 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"151\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T193206Z:209e4973-2412-46de-aeb0-3a5184e5b37d\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/1c2454d9-3f36-4903-bf4d-7d6a91e76c4f\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: DA3E32013CFF4BB48925BAB35C81373F Ref B: CHI261140504060 Ref C: 2026-04-03T19:32:05Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"209e4973-2412-46de-aeb0-3a5184e5b37d\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000ecc960\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ecca20\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kfr69r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kfr69r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012090e0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"c90c7b13-2d84-49c6-9e50-7c4041ceed5c\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 19:32:05 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"151\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T193206Z:209e4973-2412-46de-aeb0-3a5184e5b37d\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/1c2454d9-3f36-4903-bf4d-7d6a91e76c4f\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: DA3E32013CFF4BB48925BAB35C81373F Ref B: CHI261140504060 Ref C: 2026-04-03T19:32:05Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"209e4973-2412-46de-aeb0-3a5184e5b37d\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000ecc960\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f4ffd126b369fa60bcfac926d1c86cf07bd0f383435e98e40dbe70feb9bfbbce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","failed_at":"e2e","signature_id":"c193d6ca0bf762e1335e6248bbfaedb1513e9e907031106da6506fe9e3ae11b6","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000988ce0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-bx2wzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-cq5b22/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-bx2wzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-cq5b22/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000def680\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: DAC0D39B3291499EB510E196A8464291 Ref B: CHI261140504052 Ref C: 2026-04-03T19:47:34Z\",\n ],\n \"X-Ms-Request-Id\": [\n \"bbeb88a5-59d3-41b8-9d97-5ea6acb0a3e2\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"497\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 19:47:34 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/704ea369-be04-459b-a6e0-8571e30af23b\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"5d4a3adb-fb62-43e1-82b6-3016b7ae4e34\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T194735Z:5d4a3adb-fb62-43e1-82b6-3016b7ae4e34\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000988c20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000988ce0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-bx2wzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-cq5b22/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-bx2wzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-cq5b22/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000def680\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: DAC0D39B3291499EB510E196A8464291 Ref B: CHI261140504052 Ref C: 2026-04-03T19:47:34Z\",\n ],\n \"X-Ms-Request-Id\": [\n \"bbeb88a5-59d3-41b8-9d97-5ea6acb0a3e2\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"497\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 19:47:34 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/704ea369-be04-459b-a6e0-8571e30af23b\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"5d4a3adb-fb62-43e1-82b6-3016b7ae4e34\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T194735Z:5d4a3adb-fb62-43e1-82b6-3016b7ae4e34\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000988c20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-03T15:33:14Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"1440678978a31aad7415bb57db064fbb41b57ae3a02c425811281c02849884e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","failed_at":"e2e","signature_id":"0592517f8993dd48a7d28ecb3a66d91e3cacbcb2d1ba137fadd1f4b1499b1c5d","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001312be0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-lzprzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-lzprzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012997d0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 816BEB550F724B7CB369966EADF99F38 Ref B: CHI261140512042 Ref C: 2026-04-03T16:43:20Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"7a3d0639-8fbf-4c85-a9ac-cf8fd21633c5\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"126\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260403T164320Z:c9868f2c-f75d-44e0-a3df-da965aea61d0\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/36c5491e-caaa-456e-b0f7-93cb3a0d8078\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"c9868f2c-f75d-44e0-a3df-da965aea61d0\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 16:43:20 GMT\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001312a60\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"ma...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001312be0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-lzprzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-lzprzd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012997d0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 816BEB550F724B7CB369966EADF99F38 Ref B: CHI261140512042 Ref C: 2026-04-03T16:43:20Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"7a3d0639-8fbf-4c85-a9ac-cf8fd21633c5\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"126\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260403T164320Z:c9868f2c-f75d-44e0-a3df-da965aea61d0\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/36c5491e-caaa-456e-b0f7-93cb3a0d8078\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"c9868f2c-f75d-44e0-a3df-da965aea61d0\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 16:43:20 GMT\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc001312a60\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"ma...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"214797be54c49ca77e103a2b29b3f9c3eef21b289f3b19002583785c372621d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"3c8b9ed2efbc1ed0feacc60999adefd4ce91a1fb12a182af7bf5f869047f92e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","failed_at":"e2e","signature_id":"a101d653698556fe26c5748c10a638d54b4b0d1786393818b489710e7457d0e2","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a56690\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-kgcl9v\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-kgcl9v, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-kgcl9v\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-kgcl9v, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001144180\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-kgcl9v\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a56690\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-kgcl9v\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-kgcl9v, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-kgcl9v\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-kgcl9v, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001144180\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-kgcl9v\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"59a66039aae91b92666a8e131e7c0684c0c1409c738f52cdd18e56f9df80aa68","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","failed_at":"e2e","signature_id":"39b876f33ac69f73f9d738dea3e0601e180c615ee792257845cc5cbaeca39648","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775234199 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 16:36:42.47\n \"ts\"=\"2026-04-03 16:36:42.470300\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gp64gv-cl96bf\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 16:36:46.204\n STEP: creating customer resources @ 04/03/26 16:36:46.308\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 16:37:39.379\n \"ts\"=\"2026-04-03 16:37:39.379203\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gp64gv\" \"resourceGroup\"=\"rg-np-version-upgrade-gp64gv-cl96bf\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 16:53:54.034\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 17:06:34.387\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 17:10:31.018\n STEP: capturing node release images before upgrade @ 04/03/26 17:10:31.157\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 17:10:31.206\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 17:11:12.494\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775234199 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 16:36:42.47\n \"ts\"=\"2026-04-03 16:36:42.470300\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gp64gv-cl96bf\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 16:36:46.204\n STEP: creating customer resources @ 04/03/26 16:36:46.308\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 16:37:39.379\n \"ts\"=\"2026-04-03 16:37:39.379203\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gp64gv\" \"resourceGroup\"=\"rg-np-version-upgrade-gp64gv-cl96bf\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 16:53:54.034\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 17:06:34.387\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 17:10:31.018\n STEP: capturing node release images before upgrade @ 04/03/26 17:10:31.157\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 17:10:31.206\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 17:11:12.494\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"9500d5b13f741be87cb1b6e4575a399a927dfc8dc042f4090c09a99f914d1393","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","failed_at":"e2e","signature_id":"00a576e27987d92bd608669bdce0f6021d525d41d8ac0bd32cc3d4b74b822638","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ed3be0\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-k2sdw2\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-k2sdw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-k2sdw2\\\" for cluster=\\\"ea-cluster\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-k2sdw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000990240\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: FCB0395A816E4F088A6267F3F9575104 Ref B: CH1AA2020603037 Ref C: 2026-04-03T16:45:56Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"5665c40b-cd9d-496e-b953-e6902aa7df2a\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"207\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/e67cee86-8970-401d-ad15-7fe738de3eb4\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260403T164556Z:5665c40b-cd9d-496e-b953-e6902aa7df2a\",\n ],\n \"Date\": [\n \"Fri, 03 Apr 2026 16:45:56 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"1bcce103-7d8c-43c7-af0c-2c2897018d90\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000ed3b20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000ed3be0\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-k2sdw2\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-k2sdw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-k2sdw2\\\" for cluster=\\\"ea-cluster\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-k2sdw2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000990240\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: FCB0395A816E4F088A6267F3F9575104 Ref B: CH1AA2020603037 Ref C: 2026-04-03T16:45:56Z\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"5665c40b-cd9d-496e-b953-e6902aa7df2a\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Envoy-Upstream-Service-Time\": [\"207\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/e67cee86-8970-401d-ad15-7fe738de3eb4\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260403T164556Z:5665c40b-cd9d-496e-b953-e6902aa7df2a\",\n ],\n \"Date\": [\n \"Fri, 03 Apr 2026 16:45:56 GMT\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"1bcce103-7d8c-43c7-af0c-2c2897018d90\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000ed3b20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d7d2d2a417986916bb2820f40a363c8f7cf9bcabb712647131f94d237a0ceac9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","failed_at":"e2e","signature_id":"1dc9576b0674e0507d7233876890b3bd2e9f22bf38043e90a256f509b7f8b620","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"ain.beta.kubernetes.io/region\"=\"canadacentral\" \"kubernetes.io/arch\"=\"amd64\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b9c8m2q2v1t3y4d-np-1\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"canadacentral\" \"topology.kubernetes.io/zone\"=\"1\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"kubernetes.io/hostname\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-pbgf2\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-03 17:30:39.295240\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-rhnzh\" \"labels\"={\"kubernetes.io/hostname\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-rhnzh\" \"topology.kubernetes.io/zone\"=\"0\" \"failure-domain.beta.kubernetes.io/region\"=\"canadacentral\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/nodePool\"=\"b9c8m2q2v1t3y4d-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.kubernetes.io/region\"=\"canadacentral\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-03 17:30:39.295287\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-rqpwz\" \"labels\"={\"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.disk.csi.azure.com/zone\"=\"\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"kubernetes.io/os\"=\"linux\" \"node-role.kubernetes.io/worker\"=\"\" \"failure-domain.beta.kubernetes.io/region\"=\"canadacentral\" \"key2\"=\"value2\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/region\"=\"canadacentral\" \"hypershift.openshift.io/nodePool\"=\"b9c8m2q2v1t3y4d-np-1\" \"kubernetes.io/hostname\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-rqpwz\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n STEP: verifying original labels persist on 3 nodes @ 04/03/26 17:30:39.295\n STEP: verifying original taints persist on 2 nodes @ 04/03/26 17:30:39.295\n \"ts\"=\"2026-04-03 17:30:39.295520\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 17:30:39.295601\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-7gmvg5\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\"\n \"ts\"=\"2026-04-03 17:30:39.369808\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-7gmvg5\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 17:30:39.832363\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-03 17:30:39.832500\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-03 17:30:39.832588\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 17:30:39.835263\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\"\n \"ts\"=\"2026-04-03 17:30:39.835280\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\"\n \"ts\"=\"2026-04-03 17:30:39.919508\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-03 17:30:39.981371\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-7gmvg5\"\n \"ts\"=\"2026-04-03 17:30:40.314152\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 17:30:40.654016\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"msi-deployment-4\"\n \"ts\"=\"2026-04-03 17:30:40.790499\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 17:30:40.790625\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 17:30:40.790729\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nain.beta.kubernetes.io/region\"=\"canadacentral\" \"kubernetes.io/arch\"=\"amd64\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b9c8m2q2v1t3y4d-np-1\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"canadacentral\" \"topology.kubernetes.io/zone\"=\"1\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"kubernetes.io/hostname\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-pbgf2\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-03 17:30:39.295240\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-rhnzh\" \"labels\"={\"kubernetes.io/hostname\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-rhnzh\" \"topology.kubernetes.io/zone\"=\"0\" \"failure-domain.beta.kubernetes.io/region\"=\"canadacentral\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/nodePool\"=\"b9c8m2q2v1t3y4d-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.kubernetes.io/region\"=\"canadacentral\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n \"ts\"=\"2026-04-03 17:30:39.295287\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-rqpwz\" \"labels\"={\"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"hypershift.openshift.io/managed\"=\"true\" \"kubernetes.io/arch\"=\"amd64\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.disk.csi.azure.com/zone\"=\"\" \"beta.kubernetes.io/arch\"=\"amd64\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"kubernetes.io/os\"=\"linux\" \"node-role.kubernetes.io/worker\"=\"\" \"failure-domain.beta.kubernetes.io/region\"=\"canadacentral\" \"key2\"=\"value2\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.kubernetes.io/region\"=\"canadacentral\" \"hypershift.openshift.io/nodePool\"=\"b9c8m2q2v1t3y4d-np-1\" \"kubernetes.io/hostname\"=\"b9c8m2q2v1t3y4d-np-1-vfvt7-rqpwz\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n STEP: verifying original labels persist on 3 nodes @ 04/03/26 17:30:39.295\n STEP: verifying original taints persist on 2 nodes @ 04/03/26 17:30:39.295\n \"ts\"=\"2026-04-03 17:30:39.295520\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 17:30:39.295601\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-7gmvg5\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\"\n \"ts\"=\"2026-04-03 17:30:39.369808\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-7gmvg5\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 17:30:39.832363\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-03 17:30:39.832500\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n \"ts\"=\"2026-04-03 17:30:39.832588\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 17:30:39.835263\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\"\n \"ts\"=\"2026-04-03 17:30:39.835280\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\"\n \"ts\"=\"2026-04-03 17:30:39.919508\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n \"ts\"=\"2026-04-03 17:30:39.981371\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-7gmvg5\"\n \"ts\"=\"2026-04-03 17:30:40.314152\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 17:30:40.654016\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-4\" \"deployment\"=\"msi-deployment-4\"\n \"ts\"=\"2026-04-03 17:30:40.790499\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 17:30:40.790625\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 17:30:40.790729\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-6cz9nd\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-03T06:22:01Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"47ebe1bc14ea12ca96ff2fc1be54437250a25f82c15ef56d01fe0da1f7479967","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","failed_at":"e2e","signature_id":"a30104af4d916dd7cbfec3de76f31032165c563f553ec6306160c9eed3e1a2b9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d28cc0\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-cg7svw\" in resourcegroup=\"rg-stable-4-20-vkp9t8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-cg7svw in resource group rg-stable-4-20-vkp9t8, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-cg7svw\\\" in resourcegroup=\\\"rg-stable-4-20-vkp9t8\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-cg7svw in resource group rg-stable-4-20-vkp9t8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00081b460\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-cg7svw in resource group rg-stable-4-20-vkp9t8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000d28cc0\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-cg7svw\" in resourcegroup=\"rg-stable-4-20-vkp9t8\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-cg7svw in resource group rg-stable-4-20-vkp9t8, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-cg7svw\\\" in resourcegroup=\\\"rg-stable-4-20-vkp9t8\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-cg7svw in resource group rg-stable-4-20-vkp9t8, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00081b460\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-cg7svw in resource group rg-stable-4-20-vkp9t8\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"4fe23f562e840de975f4a660e21b5378fc5b89d252b153fd0f8eb8b7a0244ba9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","failed_at":"e2e","signature_id":"2d171a77f2efb9edea4c7031a4c5d4e19f877ce189da05ccb5c22454d8e0dc36","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001816030\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s8b4sw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-s8b4sw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s8b4sw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-s8b4sw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000763cd0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-s8b4sw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001816030\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-s8b4sw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-s8b4sw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-s8b4sw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-s8b4sw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000763cd0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-s8b4sw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"72ffc42d7f6a04ed11f20efca0ad3e8081e319bc895827d0cf3097f0bd85d84d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","failed_at":"e2e","signature_id":"13c51be1381e0dcdda4cd451026a1c0c513d2b1c12356428de97fbb810adac22","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00078a450\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-5wntpm\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-5h6742\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f442c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00078a450\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-5wntpm\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-5h6742\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000f442c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"7f66425f01305b64f3f95f7447c899a92aee00e180b39ddc50105f38846b33c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","failed_at":"e2e","signature_id":"9623212ee8ee787df04708ce9e32276e48e27d5b6b6f225261f64c8fa2982fe4","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006cb290\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-xw78lr\" in resourcegroup=\"rg-zstream-upgrade-4-19-xdqmmk\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-xw78lr in resource group rg-zstream-upgrade-4-19-xdqmmk, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-xw78lr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-xdqmmk\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-xw78lr in resource group rg-zstream-upgrade-4-19-xdqmmk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e85f30\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-xw78lr in resource group rg-zstream-upgrade-4-19-xdqmmk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0006cb290\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-xw78lr\" in resourcegroup=\"rg-zstream-upgrade-4-19-xdqmmk\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-xw78lr in resource group rg-zstream-upgrade-4-19-xdqmmk, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-xw78lr\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-xdqmmk\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-xw78lr in resource group rg-zstream-upgrade-4-19-xdqmmk, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000e85f30\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-xw78lr in resource group rg-zstream-upgrade-4-19-xdqmmk\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"9a78d33483d61b18666c84206a7d1d972abcb7e634cbfc87751defbeb8e71074","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","failed_at":"e2e","signature_id":"6aed24b9937e32b551ffc5f89291baa02e68d2ab1f1955dc02cb3cedc3b73787","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000bbc060\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-20-92cq46\" in resourcegroup=\"rg-zstream-upgrade-4-20-76fppt\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-92cq46 in resource group rg-zstream-upgrade-4-20-76fppt, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-92cq46\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-76fppt\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-92cq46 in resource group rg-zstream-upgrade-4-20-76fppt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00064ef70\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-92cq46 in resource group rg-zstream-upgrade-4-20-76fppt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000bbc060\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-20-92cq46\" in resourcegroup=\"rg-zstream-upgrade-4-20-76fppt\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-92cq46 in resource group rg-zstream-upgrade-4-20-76fppt, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-20-92cq46\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-20-76fppt\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-92cq46 in resource group rg-zstream-upgrade-4-20-76fppt, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00064ef70\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-20-92cq46 in resource group rg-zstream-upgrade-4-20-76fppt\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"a27397487650ddbd895d01027e713fd98a54ac08f4d20bbe41adb18d8aaf3906","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","failed_at":"e2e","signature_id":"848faab63125a0547ef1d80afb4f94d0f38adeebf4bc99e16f7dcb85fa48d55a","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a92c60\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ml5zmp-22l25f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ml5zmp/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ml5zmp-22l25f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ml5zmp/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000b47860\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/a147bbb3-95ba-46a8-99a0-b2bba5e615c9\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T074559Z:00b9e938-05fe-4cc6-bc31-bd2cf021e23c\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 195B3CCB03964C91A2E307019CCAF8FF Ref B: CH1AA2020610017 Ref C: 2026-04-03T07:45:58Z\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"94\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"00b9e938-05fe-4cc6-bc31-bd2cf021e23c\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 07:45:58 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"e6351019-162e-4755-8111-22c1271ffc29\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a92ac0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n Us...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:164]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a92c60\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ml5zmp-22l25f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ml5zmp/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ml5zmp-22l25f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ml5zmp/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000b47860\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/a147bbb3-95ba-46a8-99a0-b2bba5e615c9\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T074559Z:00b9e938-05fe-4cc6-bc31-bd2cf021e23c\",\n ],\n \"X-Msedge-Ref\": [\n \"Ref A: 195B3CCB03964C91A2E307019CCAF8FF Ref B: CH1AA2020610017 Ref C: 2026-04-03T07:45:58Z\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"94\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"00b9e938-05fe-4cc6-bc31-bd2cf021e23c\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 07:45:58 GMT\",\n ],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"e6351019-162e-4755-8111-22c1271ffc29\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a92ac0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n Us...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b5c8fbeb5c666ea137253c5842773cac38e1f9a8622fc0a51e148f3db7cae7ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","failed_at":"e2e","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"ba952582c6f2f592c3b52244f9eb73f0da7cecd2d109c654e5f40131a74e64e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","failed_at":"e2e","signature_id":"7575214c43750b29ac611602bb7070d16644923e94908c9a24d6a80a30d55228","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e00fe0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-vqx9qs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-ksxbzz/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-vqx9qs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-ksxbzz/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012d3920\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/c1c5a103-be87-4c98-bf3c-144a505f8070\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"811da485-3f04-4e6d-9e23-ab5e10af2a66\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"120\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T070704Z:811da485-3f04-4e6d-9e23-ab5e10af2a66\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 8278121BA9184CB79EDF1D3D2DC56AFF Ref B: CHI261140514036 Ref C: 2026-04-03T07:07:03Z\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"d9ee2ff2-a6a5-4e83-abb9-fd2d8ac19de7\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 07:07:03 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e00f20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e00fe0\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-vqx9qs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-ksxbzz/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-vqx9qs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-ksxbzz/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0012d3920\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/c1c5a103-be87-4c98-bf3c-144a505f8070\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"811da485-3f04-4e6d-9e23-ab5e10af2a66\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Envoy-Upstream-Service-Time\": [\"120\"],\n \"X-Ms-Routing-Request-Id\": [\n \"SWITZERLANDNORTH:20260403T070704Z:811da485-3f04-4e6d-9e23-ab5e10af2a66\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 8278121BA9184CB79EDF1D3D2DC56AFF Ref B: CHI261140514036 Ref C: 2026-04-03T07:07:03Z\",\n ],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"d9ee2ff2-a6a5-4e83-abb9-fd2d8ac19de7\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Date\": [\n \"Fri, 03 Apr 2026 07:07:03 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000e00f20\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c35bc0bd6b0dce1d40ad12e45bd1b90d8dc96dba240429b7fddf626b1c86da0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","failed_at":"e2e","signature_id":"9068b60dd134a34b45e2c4ca4bac89861a7aff0af4a58c254886eadfddc1f236","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00138aab0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4f544m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4f544m, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-4f544m\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4f544m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00074af00\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4f544m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00138aab0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-4f544m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4f544m, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-4f544m\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4f544m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00074af00\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-4f544m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-03T06:20:10Z"},"failed_at":["e2e","unknown"],"failed_test_count":11,"occurrences":[{"row_id":"001f2db4dc932633ba406134a0870d8ef098fa1a67bcad47ef879cf11f13d472","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"cc9896340c43f9cc28fd732671e71c508eb3570fc2132dd3a1db1ed99255adad","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009c8690\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-r4tzb7\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-r4tzb7, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-r4tzb7\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-r4tzb7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ade800\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-r4tzb7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009c8690\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-r4tzb7\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-r4tzb7, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-r4tzb7\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-r4tzb7, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000ade800\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-r4tzb7\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"20c0f0949bb26c5ac35cf291ea6ab2eba043ca4f5211c5b2c7469ee29cbc6121","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"a8d8ea4155e25db792b9e78559057064deaac8c45a6fd201e17b15a080b1a1dc","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012ba9f0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-v9zh48\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-v9zh48, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-v9zh48\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-v9zh48, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0006e9de0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-v9zh48\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012ba9f0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-v9zh48\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-v9zh48, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-v9zh48\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-v9zh48, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0006e9de0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-v9zh48\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"28d0e6111ef5b1248aa3e1f7833876fcea1f2b3ef2549ff0cd4e2be8514569c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"2db8f4dcd8799aa52814f81458b5d3d6bfe958d2aabd8c10d467e2405ffd0cbf","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013802d0\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-6ksz9b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-6ksz9b, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6ksz9b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-6ksz9b, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0002c2510\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-6ksz9b\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0013802d0\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-6ksz9b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-6ksz9b, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cidr-connectivity-test\\\" in resourcegroup=\\\"e2e-cidr-connectivity-6ksz9b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-6ksz9b, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0002c2510\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-6ksz9b\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"4f0cf2ac050ebf5604c3e2a3f58b48c928cbaedd2643376cb86cf5af0d3057b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"c514ce210dc57018167504b68244b55a840f03fd60d71cec6b16d5a3957651d6","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000c5f350\u003e: \n failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-gpfwx5\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-gpfwx5, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-gpfwx5\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-gpfwx5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00065e970\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-gpfwx5\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:181]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000c5f350\u003e: \n failed waiting for hcpCluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-gpfwx5\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-gpfwx5, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"ephemeral-disk\\\" in resourcegroup=\\\"ephemeral-osdisk-gpfwx5\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-gpfwx5, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00065e970\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster ephemeral-disk in resource group ephemeral-osdisk-gpfwx5\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"62119dacf03b3dd5d70ae30a8ac5b570d4f883421867d316b25e8e4710ab9eec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"e1fcf0aa66a65d43263fd7eb810b789a04625b5e903e70a78d19b18184a6fe25","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f48d80\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-c26ntd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-c26ntd, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-c26ntd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-c26ntd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0005a50c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-c26ntd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000f48d80\u003e: \n failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-c26ntd\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-c26ntd, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-c26ntd\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-c26ntd, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0005a50c0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-c26ntd\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"68b629a547e67c84e998c70c841117d4afa1ec132d514a48379ffca0742223a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"90c9cdb50ab4b9b54916bc8225453c71c231afe899393e6702c0830fcf5fda85","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008ea660\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-7gtk58\" in resourcegroup=\"rg-zstream-upgrade-4-19-sgw4tc\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-7gtk58 in resource group rg-zstream-upgrade-4-19-sgw4tc, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-7gtk58\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-sgw4tc\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-7gtk58 in resource group rg-zstream-upgrade-4-19-sgw4tc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014f3a80\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-7gtk58 in resource group rg-zstream-upgrade-4-19-sgw4tc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0008ea660\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-19-7gtk58\" in resourcegroup=\"rg-zstream-upgrade-4-19-sgw4tc\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-7gtk58 in resource group rg-zstream-upgrade-4-19-sgw4tc, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-7gtk58\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-sgw4tc\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-7gtk58 in resource group rg-zstream-upgrade-4-19-sgw4tc, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0014f3a80\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-7gtk58 in resource group rg-zstream-upgrade-4-19-sgw4tc\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"7fd42fb534a31770c742c00bff337318e587f91665b608f0a84d6fe5032d19b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"501ec29b0bb62d873bf180fc4250d995a59d71226a623c67a7cb58747dd86494","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00090f260\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-z2bmsn\" in resourcegroup=\"admin-credential-lifecycle-test-27w7lw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-z2bmsn in resource group admin-credential-lifecycle-test-27w7lw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-z2bmsn\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-27w7lw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-z2bmsn in resource group admin-credential-lifecycle-test-27w7lw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001390e00\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-z2bmsn in resource group admin-credential-lifecycle-test-27w7lw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00090f260\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-z2bmsn\" in resourcegroup=\"admin-credential-lifecycle-test-27w7lw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-z2bmsn in resource group admin-credential-lifecycle-test-27w7lw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-z2bmsn\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-27w7lw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-z2bmsn in resource group admin-credential-lifecycle-test-27w7lw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001390e00\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-z2bmsn in resource group admin-credential-lifecycle-test-27w7lw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"9d04683661982be687237131a23e88d46e7a18911bb0cfbb5ced2140fd86ed6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}},{"row_id":"c832cbd50779e513ac24fb16f2257dc4baa4da982bc9628410ee9552fbe45a97","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"f7bc02877084b7a6fc5ec919ec45196d2dced18cbc2816a4647afaedf0634f72","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775200707 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 07:18:32.861\n \"ts\"=\"2026-04-03 07:18:32.861166\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 07:18:36.582\n STEP: creating customer resources @ 04/03/26 07:18:36.663\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 07:19:30.275\n \"ts\"=\"2026-04-03 07:19:30.276042\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-n644j9\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 07:32:30.616\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 07:43:09.56\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 07:50:07.133\n STEP: capturing node release images before upgrade @ 04/03/26 07:50:07.268\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 07:50:07.316\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 07:51:18.942\n STEP: verifying node pool GET still reflects the new version @ 04/03/26 08:21:21.401\n \"ts\"=\"2026-04-03 08:21:21.916429\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 08:21:21.916513\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-n644j9-tzw4ms\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\n \"ts\"=\"2026-04-03 08:21:22.021919\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-n644j9-tzw4ms\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 08:21:22.381506\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-03 08:21:22.381609\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-03 08:21:22.381674\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 08:21:22.383217\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-5\"\n \"ts\"=\"2026-04-03 08:21:22.383231\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\n \"ts\"=\"2026-04-03 08:21:22.482394\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\" \"deployment\"=\"pooledNonMsiScopedAssignments-ohfyjjvkpxkig\"\n \"ts\"=\"2026-04-03 08:21:22.529060\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-5\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 08:21:22.548926\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-n644j9-tzw4ms\"\n \"ts\"=\"2026-04-03 08:21:22.689884\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-5\" \"deployment\"=\"msi-deployment-5\"\n \"ts\"=\"2026-04-03 08:21:22.822113\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 08:21:22.822251\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 08:21:22.822398\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775200707 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 07:18:32.861\n \"ts\"=\"2026-04-03 07:18:32.861166\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 07:18:36.582\n STEP: creating customer resources @ 04/03/26 07:18:36.663\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 07:19:30.275\n \"ts\"=\"2026-04-03 07:19:30.276042\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-n644j9\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 07:32:30.616\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 07:43:09.56\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 07:50:07.133\n STEP: capturing node release images before upgrade @ 04/03/26 07:50:07.268\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 07:50:07.316\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 07:51:18.942\n STEP: verifying node pool GET still reflects the new version @ 04/03/26 08:21:21.401\n \"ts\"=\"2026-04-03 08:21:21.916429\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 08:21:21.916513\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-n644j9-tzw4ms\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\n \"ts\"=\"2026-04-03 08:21:22.021919\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-n644j9-tzw4ms\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 08:21:22.381506\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-03 08:21:22.381609\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-03 08:21:22.381674\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 08:21:22.383217\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-5\"\n \"ts\"=\"2026-04-03 08:21:22.383231\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\n \"ts\"=\"2026-04-03 08:21:22.482394\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\" \"deployment\"=\"pooledNonMsiScopedAssignments-ohfyjjvkpxkig\"\n \"ts\"=\"2026-04-03 08:21:22.529060\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-5\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 08:21:22.548926\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-n644j9-tzw4ms\"\n \"ts\"=\"2026-04-03 08:21:22.689884\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-5\" \"deployment\"=\"msi-deployment-5\"\n \"ts\"=\"2026-04-03 08:21:22.822113\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 08:21:22.822251\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 08:21:22.822398\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-n644j9-67j8pv\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command"}},{"row_id":"e92fd6bc0632e0320ba3702b87faa11f8f22947564d75d933176b892f78f8461","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"e2e","signature_id":"1798b4cb1a37816e867fca622bd7878502beca3293e01e7628bd7340f11a75c8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00106d830\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-bqjj6q\" in resourcegroup=\"rg-stable-4-20-5xlww9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-bqjj6q in resource group rg-stable-4-20-5xlww9, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-bqjj6q\\\" in resourcegroup=\\\"rg-stable-4-20-5xlww9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-bqjj6q in resource group rg-stable-4-20-5xlww9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001241df0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-bqjj6q in resource group rg-stable-4-20-5xlww9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:125]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00106d830\u003e: \n failed waiting for hcpCluster=\"cluster-stable-4-20-bqjj6q\" in resourcegroup=\"rg-stable-4-20-5xlww9\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-bqjj6q in resource group rg-stable-4-20-5xlww9, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-stable-4-20-bqjj6q\\\" in resourcegroup=\\\"rg-stable-4-20-5xlww9\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-bqjj6q in resource group rg-stable-4-20-5xlww9, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001241df0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-stable-4-20-bqjj6q in resource group rg-stable-4-20-5xlww9\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded"}},{"row_id":"f80a88fa353acdb6de033257219e29d7426bd67d66264b1e914839cb25270045","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","failed_at":"unknown","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","test_name":"Job run should complete before timeout","test_suite":"prowjob-junit","failure_text":"{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-03T08:22:36Z\"}","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039885780922404864","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-04-03T02:00:44Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fec4d9c49f48e1afd54e7dcf03c691a810c2c246ba579f8951a5c07377fd2632","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039885780922404864","occurred_at":"2026-04-03T02:00:44Z","failed_at":"e2e","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","test_suite":"prod/parallel","failure_text":"fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}