{"meta":{"start_date":"2026-04-10","end_date":"2026-04-10","start_at":"2026-04-10T00:00:00Z","end_at":"2026-04-11T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T17:16:35Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":11,"failed_runs":9,"raw_occurrences":73,"matched_occurrences":73,"runs_affected":9},"rows":[{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","occurrences":25,"runs_affected":5,"failed_runs":5,"run_impact_percent":45.45454545454545,"anchor_occurrences":25,"after_last_push_count":93,"trend_counts":[0,3,1,16,0,19,25],"trend_range":"2026-04-04..2026-04-10 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":28,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 12:03:04.020611\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:880\"\n \"ts\"=\"2026-04-10 12:03:04.028462\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 12:03:04.028\n \"ts\"=\"2026-04-10 12:03:04.028545\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 12:03:08.089\nW0410 12:03:08.168105 880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/10/26 12:03:08.168\n STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 12:04:03.752\n \"ts\"=\"2026-04-10 12:04:03.752222\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-wdq4gs\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:15:44.21\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:06.745\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:05.032\nW0410 12:29:05.158022 880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: capturing node release images before upgrade @ 04/10/26 12:29:05.158\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 12:29:05.194\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:48.069\n STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:49.726\n \"ts\"=\"2026-04-10 12:51:50.360556\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:51:50.360655\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 12:51:50.487211\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:51:51.762518\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:51.762617\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:51.762676\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:51.764204\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\"\n \"ts\"=\"2026-04-10 12:51:51.764225\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 12:51:51.859754\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"pooledNonMsiScopedAssignments-dpbn3ikyauobc\"\n \"ts\"=\"2026-04-10 12:51:52.188516\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\"\n \"ts\"=\"2026-04-10 12:51:52.242487\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:51:52.779918\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"msi-deployment-131\"\n \"ts\"=\"2026-04-10 12:51:52.917289\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:52.917406\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:51:52.917517\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 13:02:10.173344\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 13:02:10.173371\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nAPI server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 12:03:04.020611\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:880\"\n \"ts\"=\"2026-04-10 12:03:04.028462\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 12:03:04.028\n \"ts\"=\"2026-04-10 12:03:04.028545\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 12:03:08.089\nW0410 12:03:08.168105 880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/10/26 12:03:08.168\n STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 12:04:03.752\n \"ts\"=\"2026-04-10 12:04:03.752222\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-wdq4gs\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:15:44.21\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:06.745\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:05.032\nW0410 12:29:05.158022 880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: capturing node release images before upgrade @ 04/10/26 12:29:05.158\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 12:29:05.194\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:48.069\n STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:49.726\n \"ts\"=\"2026-04-10 12:51:50.360556\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:51:50.360655\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 12:51:50.487211\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:51:51.762518\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:51.762617\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:51.762676\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:51.764204\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\"\n \"ts\"=\"2026-04-10 12:51:51.764225\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 12:51:51.859754\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"pooledNonMsiScopedAssignments-dpbn3ikyauobc\"\n \"ts\"=\"2026-04-10 12:51:52.188516\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\"\n \"ts\"=\"2026-04-10 12:51:52.242487\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:51:52.779918\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"msi-deployment-131\"\n \"ts\"=\"2026-04-10 12:51:52.917289\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:52.917406\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:51:52.917517\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 13:02:10.173344\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 13:02:10.173371\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775823245 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n \"ts\"=\"2026-04-10 12:14:05.070742\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:919\"\n \"ts\"=\"2026-04-10 12:14:05.079100\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/10/26 12:14:05.079\n \"ts\"=\"2026-04-10 12:14:05.079151\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n STEP: creating cluster parameters @ 04/10/26 12:14:09.56\n STEP: creating customer resources @ 04/10/26 12:14:09.56\n STEP: creating the HCP cluster @ 04/10/26 12:14:54.459\n \"ts\"=\"2026-04-10 12:14:54.459322\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n STEP: creating a storage account for shoebox logs @ 04/10/26 12:29:05.918\n \"ts\"=\"2026-04-10 12:29:25.513737\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxfpqdwk\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7pqpv2/providers/Microsoft.Storage/storageAccounts/shoeboxfpqdwk\"\n STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 12:29:25.513\n \"ts\"=\"2026-04-10 12:29:30.702291\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n STEP: waiting for log containers to appear in the storage account @ 04/10/26 12:29:30.702\n \"ts\"=\"2026-04-10 12:55:30.837118\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxfpqdwk\" \"timestamp\"=\"2026-04-10T12:55:30Z\"\n \"ts\"=\"2026-04-10 12:55:30.837292\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:55:30.837387\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n \"ts\"=\"2026-04-10 12:55:30.988434\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:55:32.339790\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-10 12:55:32.339933\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-10 12:55:32.340053\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:55:32.341960\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\"\n \"ts\"=\"2026-04-10 12:55:32.342015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n \"ts\"=\"2026-04-10 12:55:32.680479\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n \"ts\"=\"2026-04-10 12:55:32.749543\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\"\n \"ts\"=\"2026-04-10 12:55:32.952168\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:55:33.567562\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"msi-deployment-95\"\n \"ts\"=\"2026-04-10 12:55:33.845873\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:55:33.846012\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:55:33.846126\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775823245 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n \"ts\"=\"2026-04-10 12:14:05.070742\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:919\"\n \"ts\"=\"2026-04-10 12:14:05.079100\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/10/26 12:14:05.079\n \"ts\"=\"2026-04-10 12:14:05.079151\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n STEP: creating cluster parameters @ 04/10/26 12:14:09.56\n STEP: creating customer resources @ 04/10/26 12:14:09.56\n STEP: creating the HCP cluster @ 04/10/26 12:14:54.459\n \"ts\"=\"2026-04-10 12:14:54.459322\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n STEP: creating a storage account for shoebox logs @ 04/10/26 12:29:05.918\n \"ts\"=\"2026-04-10 12:29:25.513737\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxfpqdwk\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7pqpv2/providers/Microsoft.Storage/storageAccounts/shoeboxfpqdwk\"\n STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 12:29:25.513\n \"ts\"=\"2026-04-10 12:29:30.702291\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n STEP: waiting for log containers to appear in the storage account @ 04/10/26 12:29:30.702\n \"ts\"=\"2026-04-10 12:55:30.837118\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxfpqdwk\" \"timestamp\"=\"2026-04-10T12:55:30Z\"\n \"ts\"=\"2026-04-10 12:55:30.837292\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:55:30.837387\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n \"ts\"=\"2026-04-10 12:55:30.988434\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:55:32.339790\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-10 12:55:32.339933\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-10 12:55:32.340053\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:55:32.341960\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\"\n \"ts\"=\"2026-04-10 12:55:32.342015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n \"ts\"=\"2026-04-10 12:55:32.680479\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n \"ts\"=\"2026-04-10 12:55:32.749543\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\"\n \"ts\"=\"2026-04-10 12:55:32.952168\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:55:33.567562\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"msi-deployment-95\"\n \"ts\"=\"2026-04-10 12:55:33.845873\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:55:33.846012\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:55:33.846126\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","t/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\nW0410 12:04:14.665210 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 12:04:14.748424 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 12:04:14.748740\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:893\"\n \"ts\"=\"2026-04-10 12:04:14.933427\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 12:04:14.933\n \"ts\"=\"2026-04-10 12:04:14.933646\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 12:04:19.473\n STEP: creating customer resources @ 04/10/26 12:04:19.579\n STEP: creating the HCP cluster with version 4.20.17 @ 04/10/26 12:05:04.201\n \"ts\"=\"2026-04-10 12:05:04.201160\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-jnm8s9\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:16:02.227\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:01.442\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:09.867\n STEP: capturing node release images before upgrade @ 04/10/26 12:29:10.041\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/10/26 12:29:10.08\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:52.504\n STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:54.086\n \"ts\"=\"2026-04-10 12:51:54.286433\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:51:54.286512\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 12:51:54.379693\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:51:55.257214\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:55.257316\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:55.257391\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:55.259038\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\"\n \"ts\"=\"2026-04-10 12:51:55.259056\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 12:51:55.353380\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"pooledNonMsiScopedAssignments-jfl6npesxtst4\"\n \"ts\"=\"2026-04-10 12:51:55.415329\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\"\n \"ts\"=\"2026-04-10 12:51:55.578837\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:51:55.709848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"msi-deployment-129\"\n \"ts\"=\"2026-04-10 12:51:55.860303\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:55.860430\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:51:55.860601\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 13:02:14.746040\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 13:02:14.746065\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\nW0410 12:04:14.665210 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 12:04:14.748424 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 12:04:14.748740\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:893\"\n \"ts\"=\"2026-04-10 12:04:14.933427\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 12:04:14.933\n \"ts\"=\"2026-04-10 12:04:14.933646\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 12:04:19.473\n STEP: creating customer resources @ 04/10/26 12:04:19.579\n STEP: creating the HCP cluster with version 4.20.17 @ 04/10/26 12:05:04.201\n \"ts\"=\"2026-04-10 12:05:04.201160\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-jnm8s9\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:16:02.227\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:01.442\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:09.867\n STEP: capturing node release images before upgrade @ 04/10/26 12:29:10.041\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/10/26 12:29:10.08\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:52.504\n STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:54.086\n \"ts\"=\"2026-04-10 12:51:54.286433\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:51:54.286512\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 12:51:54.379693\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:51:55.257214\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:55.257316\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:55.257391\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:55.259038\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\"\n \"ts\"=\"2026-04-10 12:51:55.259056\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 12:51:55.353380\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"pooledNonMsiScopedAssignments-jfl6npesxtst4\"\n \"ts\"=\"2026-04-10 12:51:55.415329\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\"\n \"ts\"=\"2026-04-10 12:51:55.578837\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:51:55.709848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"msi-deployment-129\"\n \"ts\"=\"2026-04-10 12:51:55.860303\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:55.860430\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:51:55.860601\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 13:02:14.746040\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 13:02:14.746065\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"1656f2b598b891e3f588923f365c70c85887da29099479a61fa374a63b758405","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"7ba7fc6de5bcafc9ea168179205860c649c276fe4162534ff6f785c88ed1a1bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"e115f0d91ca258d05c604dcc10d976dfb30b4ee3c98378e747983d1d19c90761","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"4b59ed1e7bc1c9174eb070f50a729e281f859e483dcfa086bc53ac0d3016c8bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"b052f95c7ac43877358d9b58edf1d1cf79d94d9b5351ee84ab0314fd489ecd4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"d24344800d201228d64db010e816a5e25cf74acb86cc4c5d05002723bf4f2156","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"e2072d0c469b46a1d7f7d3e238ea467288fa9ef089199ab06d9636c4c900cdef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"e5e871a9a2c633cc721fe9ca827b4448e9e818af940c191290f0b5f0ee7e369f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042523562169864192","occurred_at":"2026-04-10T08:42:20Z","signature_id":"977c5a79955b703d5d52fa873359cbadc299b6ed68168fe22a6de3d1a85024e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"1b3bdf86f9c812b2a7b8f5c8ab5b543a1b2a695fcf36e4c92a1990c7902ccee1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"774c8b9425c53dec4fa564c04d6b0b55f586c357fb5487ed255e12937b8edcaf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"c269ac0dc1e4722962bb2c16446c66884aff962620fc94e198f5797dd1547de2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"eb113763ca9eb39cbfbf3f6d9b2910a4d656864f89ff1075609a1e9aa9129dd1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"456956c00d2cd03fe0e688f32f7d08a0fab5648b265b6120efc3254d3963f16b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"711813ff0ef173ae4c74b1746c0d4883d500104597abcceb58cf5fd7e1fa6fb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"74a2f372f73f571cf44d910b50c9e1559c9dfb4de2568689b9ffb24d2fdb9485","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"7a3430cc1684e21012eb2689e5bed9d9e16bc291639be081252556bdf6682079","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"85b95c9895e4e0f7c2fd44a5c6864b35807d9f8a75a3b9bb1666804bfe472133","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"a09d0db5dedab2fb189aca9a3aad2190a32644a96b9529a61aa4af99f28d3beb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"a4e2d1c4ce37762825496b9afd8965783d6fd32cc144b275c851da1948e6b75e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"ae1f25418990f1a3fcdf9ce89f3756ee2927b16ebcbdfc62e13edc53b8bfa567","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"b7bb55f3055dea91e4e2e0a35768691dd8a3bb6770c7125349f05399ef297db8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"c79c0556df8ad3acfeb27115565aa651ad5885ac90240887372b6c08276ea8a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"cb1103453558f656c64bcf405755d3e4078107305cbaf3d067fbc78e283e95ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"ccf1c285155e7641c812ec1a64a480027fe0f5935acb52b42c0d556cedc15270","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","occurrences":18,"runs_affected":4,"failed_runs":4,"run_impact_percent":36.36363636363637,"anchor_occurrences":18,"after_last_push_count":117,"trend_counts":[0,21,1,10,0,22,18],"trend_range":"2026-04-04..2026-04-10 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":20,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:116]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012c4a20\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-2xwn4s\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-2xwn4s, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:116]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012c4a20\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-2xwn4s\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster cilium-cluster in resource group e2e-cilium-2xwn4s, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00156eb10\u003e: \n failed to create HCP cluster cluster-zstream-4-19-6mnfl6, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6mnfl6 in resource group rg-zstream-upgrade-4-19-bjj5n4, error: failed waiting for cluster=\"cluster-zstream-4-19-6mnfl6\" in resourcegroup=\"rg-zstream-upgrade-4-19-bjj5n4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6mnfl6 in resource group rg-zstream-upgrade-4-19-bjj5n4, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:141]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00156eb10\u003e: \n failed to create HCP cluster cluster-zstream-4-19-6mnfl6, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6mnfl6 in resource group rg-zstream-upgrade-4-19-bjj5n4, error: failed waiting for cluster=\"cluster-zstream-4-19-6mnfl6\" in resourcegroup=\"rg-zstream-upgrade-4-19-bjj5n4\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cluster-zstream-4-19-6mnfl6 in resource group rg-zstream-upgrade-4-19-bjj5n4, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015c05a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cw77vd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cw77vd, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:108]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015c05a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-cw77vd\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-cw77vd, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"4417f9ebee113ede569a63481c296343013ef4c0dd63cb13ca927a1c9d0575a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"f4a2a227d30c0371e3cd0edc206bb57bb934f7573ea663644af9bd1f992bdc4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"f9d8e75397a206a6268f651b0be6637d057a861d0bbf464e067b67c82a1421dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"0d0065a85884f57dc55146b3cd6f9e7f652f0498eab4c9f7cd357e374c69da98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"3342dc667feb6683fa8d37b7adb8bf7cebbf78de37e827acc0e3826580e28e1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"3acbfe85d42d4b2af0028ce47dd46b2ecf6328bee6e74b263673d44b4664b09c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"3c74c2b7b0d98970008636e4702e3dafe55cd39f4902e823fddd1b1caeaf1437","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"466a1bc425e7596e811a1a5b201d5b80622e0baf5fe5f127acd6d33a30be9d6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"7c5da14974f842e2f7863d15e9875fdc09997be97e160621aa4a3cd853a7b438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"a8c8fd1a5a8ee69127b99cd5468e4422b9f95af227abcd6b8f9e4da53d12eddf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"0591505a8d0d09ee27d773b578770c0879cc0fd3c1615c87b0b878047697e942","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"633be0bfad8cef28de804331d1e9aee9b444f7683897f6917a5919e6188f7c6c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"685faef7c56cc1e714facd7df48e1f0acf7fc3632e9fd34b791af4ada2983c56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"79e17694bc7eb6852bb1fb17ca10ba1644573ee030a39c7db08020c8c4393908","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"d4beb9840975059b78ebb2bbdc1f3bd005e206d72191c83051709934b2acc299","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"ef4ac3a0886e7bfd19c86141172c407d907b090c56c349762c5baab5d47ea58a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"a1ffbe285ac5025d1bccac53237d64f36796874c70d5c5ae5d58b795494658ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"ee3c35e66876b8446d35f42a776892654f3b6ea66fb1b4ab7ae0775c93d7a8c5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":15,"runs_affected":5,"failed_runs":5,"run_impact_percent":45.45454545454545,"anchor_occurrences":15,"after_last_push_count":74,"trend_counts":[0,0,0,14,0,4,15],"trend_range":"2026-04-04..2026-04-10 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-03-30"],"prior_runs_affected":18,"prior_last_seen_at":"2026-04-03T18:45:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e73080\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e73080\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007fd2f0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mhtwts\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-mhtwts, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0007fd2f0\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mhtwts\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-tags-cluster in resource group patch-tags-mhtwts, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dfc0f0\u003e: \n failed waiting for hcpCluster=\"cluster-candidate-4-20-vl26hx\" in resourcegroup=\"rg-candidate-4-20-lbfbj4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-20-vl26hx in resource group rg-candidate-4-20-lbfbj4, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:129]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dfc0f0\u003e: \n failed waiting for hcpCluster=\"cluster-candidate-4-20-vl26hx\" in resourcegroup=\"rg-candidate-4-20-lbfbj4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-candidate-4-20-vl26hx in resource group rg-candidate-4-20-lbfbj4, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"2e4c68add548aaa6737eadeda2aefb454957e84c997a068f440595794fe5ea8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"324ffdaba18d3147faf9a4285ad6fa60df88e7720fcd0b876adf23da5ffa8578","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"ad1a784b6233c088bf1cbcdab05052ce4206e83faaf04a30cff15ea5dc427b03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042684774031560704","occurred_at":"2026-04-10T19:22:53Z","signature_id":"a9d92a762e487ff6e17f5d7d33bf21ab33f872b713827d1508be3bd3f1e9a462","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"094a6c2314265f60b76e8fcbe84a2c8e44259e101a52c5a4c142e5713aec8804","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"5a88ff7c5b00148eaccf0a7f4dea26b63c8ec41eaceae91a95ba6b5e42525ddc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"7a0ad282eff611158e8e43418bc514839cbe0023046786a1ce9aab3f8e0a2e7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"97ed143ef45d8dc14377b681c99ace8f580e81f564813b645e8377e8d924dee7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"9c9530d101e12ab5a650c8fc6e9715606e353ecc95c0653e2075234a94e464fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"e1e9672e98e7c5f28b1706f8fa646f2740d0999ee129fbdcef9034e7e05cce00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"a87063a43d794d60b0e265445eae9710463acd30489634d881a98f9be587a1a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"e72458e45371568f1114245b29ad35b4f5307ce068bdec27027c68de7680c7db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"5012567a501f3d37567b8d2827f0fd6d46f0b92140488b498409244711ec15f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"ba350b37506f60e30bb4bb27da0434f337a2464ab0c305b5ebf8b0a627f56e6c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"de4a5ce3ac3c5e9aea9290d16182f031cb6e3c37df7abdcce982c39a9f3fc408","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":8,"runs_affected":4,"failed_runs":4,"run_impact_percent":36.36363636363637,"anchor_occurrences":8,"after_last_push_count":32,"trend_counts":[0,3,1,1,0,5,8],"trend_range":"2026-04-04..2026-04-10 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-30"],"prior_runs_affected":14,"prior_last_seen_at":"2026-04-05T20:06:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:239]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c3a700\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9hc929/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:239]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000c3a700\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9hc929/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/oidc-wi-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e90120\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-rn26mb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:170]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e90120\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-rn26mb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001312300\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-26bkkw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-phh9x4/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001312300\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-19-26bkkw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-19-phh9x4/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"55ef6ef4d9ea61f11d4d23769864db3f0d507685dc0ef554cb5890b869bcca8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"1b2c3288fde7d0baf2d757029c5c1cfe45b7c15828e1e4ac53ffd4d897080b18","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"3f552bc1f85dd1b4bd1189c16824c26c1b464f002c113d93d56f72f56c0dcc27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"e09f5da2799329b5ee78a500dfa49b480e4ee88c92315c4c097a19512e99ec74","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"f726f5cdc0a97832459d0a83f77b5fc8d60caabf1f5995ed8b497222589a4f3d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"8b1222a8560e2917674611da65a4169d2346b2401f470a0385b6c34c7da27c8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"3d519ffa44179193aedca896e5792ca8b7df0696ac0a7f979d1d282a5702f1c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"f2fc8aa99648d2d62ceefe9d3e42b8eb17da274af35f704f60c14515a7ecceb4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":45.45454545454545,"anchor_occurrences":5,"after_last_push_count":21,"trend_counts":[0,1,1,5,0,3,5],"trend_range":"2026-04-04..2026-04-10 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-03-30"],"prior_runs_affected":21,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":5}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T13:03:33Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T13:01:55Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T10:44:38Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042523562169864192","occurred_at":"2026-04-10T08:42:20Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,1,0,0,0,0,1],"trend_range":"2026-04-04..2026-04-10 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-16","2026-03-30"],"prior_runs_affected":4,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"8a441c3cc9a1724fd565ecc68f397cdfdb078c056f41b0ab5661bd4e719eb328","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4acfe441005f8003e7187d8bb50b5a3c1645ce7924b7f2de03ef6bdb2cb05e18","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Cluster '\u003cid\u003e' has an external authentication pending deletion; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":9.090909090909092,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-04-04..2026-04-10 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:294]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001508c80\u003e: \n failed creating external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-n5j64n\" for cluster=\"cidr-connectivity-test\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n5j64n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/externalAuths/ext-auth-cidr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Cluster '2pj8n2ov52hfcbj1lfepdkk77i8o48uj' has an external authentication pending deletion\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:294]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001508c80\u003e: \n failed creating external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-n5j64n\" for cluster=\"cidr-connectivity-test\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n5j64n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/externalAuths/ext-auth-cidr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Cluster '2pj8n2ov52hfcbj1lfepdkk77i8o48uj' has an external authentication pending deletion\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"71b0ec01edd9207bb394cba62c96fa66ee1e1f320a74672b8a730144deb83883","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}