{"meta":{"start_date":"2026-04-06","end_date":"2026-04-12","start_at":"2026-04-06T00:00:00Z","end_at":"2026-04-13T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T14:54:29Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":41,"failed_runs":31,"raw_occurrences":234,"matched_occurrences":233,"runs_affected":30},"rows":[{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":61,"runs_affected":15,"failed_runs":15,"run_impact_percent":36.58536585365854,"anchor_occurrences":61,"after_last_push_count":93,"trend_counts":[3,1,16,0,19,25,0,0],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":28,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":7},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":9},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","occurrences":5},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 12:03:04.020611\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:880\"\n \"ts\"=\"2026-04-10 12:03:04.028462\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 12:03:04.028\n \"ts\"=\"2026-04-10 12:03:04.028545\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 12:03:08.089\nW0410 12:03:08.168105 880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/10/26 12:03:08.168\n STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 12:04:03.752\n \"ts\"=\"2026-04-10 12:04:03.752222\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-wdq4gs\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:15:44.21\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:06.745\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:05.032\nW0410 12:29:05.158022 880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: capturing node release images before upgrade @ 04/10/26 12:29:05.158\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 12:29:05.194\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:48.069\n STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:49.726\n \"ts\"=\"2026-04-10 12:51:50.360556\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:51:50.360655\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 12:51:50.487211\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:51:51.762518\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:51.762617\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:51.762676\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:51.764204\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\"\n \"ts\"=\"2026-04-10 12:51:51.764225\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 12:51:51.859754\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"pooledNonMsiScopedAssignments-dpbn3ikyauobc\"\n \"ts\"=\"2026-04-10 12:51:52.188516\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\"\n \"ts\"=\"2026-04-10 12:51:52.242487\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:51:52.779918\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"msi-deployment-131\"\n \"ts\"=\"2026-04-10 12:51:52.917289\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:52.917406\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:51:52.917517\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 13:02:10.173344\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 13:02:10.173371\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nAPI server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 12:03:04.020611\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.21.zLatest|pid:880\"\n \"ts\"=\"2026-04-10 12:03:04.028462\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 12:03:04.028\n \"ts\"=\"2026-04-10 12:03:04.028545\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 12:03:08.089\nW0410 12:03:08.168105 880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: creating customer resources @ 04/10/26 12:03:08.168\n STEP: creating the HCP cluster with version 4.21.7 @ 04/10/26 12:04:03.752\n \"ts\"=\"2026-04-10 12:04:03.752222\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-wdq4gs\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:15:44.21\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:06.745\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:05.032\nW0410 12:29:05.158022 880 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n STEP: capturing node release images before upgrade @ 04/10/26 12:29:05.158\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/10/26 12:29:05.194\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:48.069\n STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:49.726\n \"ts\"=\"2026-04-10 12:51:50.360556\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:51:50.360655\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 12:51:50.487211\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-wdq4gs-95qms7\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:51:51.762518\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:51.762617\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:51.762676\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:51.764204\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\"\n \"ts\"=\"2026-04-10 12:51:51.764225\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 12:51:51.859754\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"pooledNonMsiScopedAssignments-dpbn3ikyauobc\"\n \"ts\"=\"2026-04-10 12:51:52.188516\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-wdq4gs-95qms7\"\n \"ts\"=\"2026-04-10 12:51:52.242487\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:51:52.779918\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-131\" \"deployment\"=\"msi-deployment-131\"\n \"ts\"=\"2026-04-10 12:51:52.917289\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:52.917406\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:51:52.917517\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 13:02:10.173344\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\n \"ts\"=\"2026-04-10 13:02:10.173371\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-wdq4gs-qwpw89\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775823245 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n \"ts\"=\"2026-04-10 12:14:05.070742\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:919\"\n \"ts\"=\"2026-04-10 12:14:05.079100\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/10/26 12:14:05.079\n \"ts\"=\"2026-04-10 12:14:05.079151\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n STEP: creating cluster parameters @ 04/10/26 12:14:09.56\n STEP: creating customer resources @ 04/10/26 12:14:09.56\n STEP: creating the HCP cluster @ 04/10/26 12:14:54.459\n \"ts\"=\"2026-04-10 12:14:54.459322\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n STEP: creating a storage account for shoebox logs @ 04/10/26 12:29:05.918\n \"ts\"=\"2026-04-10 12:29:25.513737\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxfpqdwk\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7pqpv2/providers/Microsoft.Storage/storageAccounts/shoeboxfpqdwk\"\n STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 12:29:25.513\n \"ts\"=\"2026-04-10 12:29:30.702291\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n STEP: waiting for log containers to appear in the storage account @ 04/10/26 12:29:30.702\n \"ts\"=\"2026-04-10 12:55:30.837118\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxfpqdwk\" \"timestamp\"=\"2026-04-10T12:55:30Z\"\n \"ts\"=\"2026-04-10 12:55:30.837292\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:55:30.837387\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n \"ts\"=\"2026-04-10 12:55:30.988434\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:55:32.339790\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-10 12:55:32.339933\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-10 12:55:32.340053\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:55:32.341960\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\"\n \"ts\"=\"2026-04-10 12:55:32.342015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n \"ts\"=\"2026-04-10 12:55:32.680479\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n \"ts\"=\"2026-04-10 12:55:32.749543\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\"\n \"ts\"=\"2026-04-10 12:55:32.952168\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:55:33.567562\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"msi-deployment-95\"\n \"ts\"=\"2026-04-10 12:55:33.845873\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:55:33.846012\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:55:33.846126\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775823245 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, Stage-And-Prod-Only]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:38\n \"ts\"=\"2026-04-10 12:14:05.070742\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-be-able-to-forward-control-plane-logs-to-a-storage-account-via-shoebox-diagnostic-settings|pid:919\"\n \"ts\"=\"2026-04-10 12:14:05.079100\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating a resource group @ 04/10/26 12:14:05.079\n \"ts\"=\"2026-04-10 12:14:05.079151\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n STEP: creating cluster parameters @ 04/10/26 12:14:09.56\n STEP: creating customer resources @ 04/10/26 12:14:09.56\n STEP: creating the HCP cluster @ 04/10/26 12:14:54.459\n \"ts\"=\"2026-04-10 12:14:54.459322\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"shoebox-hcp-cluster\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n STEP: creating a storage account for shoebox logs @ 04/10/26 12:29:05.918\n \"ts\"=\"2026-04-10 12:29:25.513737\" \"level\"=0 \"msg\"=\"storage account created\" \"name\"=\"shoeboxfpqdwk\" \"id\"=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7pqpv2/providers/Microsoft.Storage/storageAccounts/shoeboxfpqdwk\"\n STEP: enabling diagnostic settings on the HCP cluster @ 04/10/26 12:29:25.513\n \"ts\"=\"2026-04-10 12:29:30.702291\" \"level\"=0 \"msg\"=\"diagnostic setting created\" \"name\"=\"shoebox-diag-setting\"\n STEP: waiting for log containers to appear in the storage account @ 04/10/26 12:29:30.702\n \"ts\"=\"2026-04-10 12:55:30.837118\" \"level\"=0 \"msg\"=\"shoebox log containers found in storage account\" \"storageAccount\"=\"shoeboxfpqdwk\" \"timestamp\"=\"2026-04-10T12:55:30Z\"\n \"ts\"=\"2026-04-10 12:55:30.837292\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:55:30.837387\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n \"ts\"=\"2026-04-10 12:55:30.988434\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-shoebox-hcp-cluster-j9gxkf\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:55:32.339790\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-10 12:55:32.339933\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-28def4d2520d754dc684cda51e7fbbbda7eafe4c1b56787244f7f5c8f8c5f740.yaml.gz\"\n \"ts\"=\"2026-04-10 12:55:32.340053\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:55:32.341960\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\"\n \"ts\"=\"2026-04-10 12:55:32.342015\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\n \"ts\"=\"2026-04-10 12:55:32.680479\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"pooledNonMsiScopedAssignments-22eudp3sazqeq\"\n \"ts\"=\"2026-04-10 12:55:32.749543\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\" \"deployment\"=\"customer-infra-shoebox-hcp-cluster-j9gxkf\"\n \"ts\"=\"2026-04-10 12:55:32.952168\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:55:33.567562\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-95\" \"deployment\"=\"msi-deployment-95\"\n \"ts\"=\"2026-04-10 12:55:33.845873\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:55:33.846012\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:55:33.846126\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"shoebox-cluster-7pqpv2\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","t/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\nW0410 12:04:14.665210 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 12:04:14.748424 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 12:04:14.748740\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:893\"\n \"ts\"=\"2026-04-10 12:04:14.933427\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 12:04:14.933\n \"ts\"=\"2026-04-10 12:04:14.933646\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 12:04:19.473\n STEP: creating customer resources @ 04/10/26 12:04:19.579\n STEP: creating the HCP cluster with version 4.20.17 @ 04/10/26 12:05:04.201\n \"ts\"=\"2026-04-10 12:05:04.201160\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-jnm8s9\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:16:02.227\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:01.442\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:09.867\n STEP: capturing node release images before upgrade @ 04/10/26 12:29:10.041\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/10/26 12:29:10.08\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:52.504\n STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:54.086\n \"ts\"=\"2026-04-10 12:51:54.286433\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:51:54.286512\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 12:51:54.379693\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:51:55.257214\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:55.257316\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:55.257391\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:55.259038\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\"\n \"ts\"=\"2026-04-10 12:51:55.259056\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 12:51:55.353380\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"pooledNonMsiScopedAssignments-jfl6npesxtst4\"\n \"ts\"=\"2026-04-10 12:51:55.415329\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\"\n \"ts\"=\"2026-04-10 12:51:55.578837\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:51:55.709848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"msi-deployment-129\"\n \"ts\"=\"2026-04-10 12:51:55.860303\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:55.860430\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:51:55.860601\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 13:02:14.746040\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 13:02:14.746065\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:226\nW0410 12:04:14.665210 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\nW0410 12:04:14.748424 893 cincinnati.go:308] Conditional update to 4.21.8, risk \"PrecisionTimeProtocolDPLLPins\", has empty pruned matchingRules; dropping this target to avoid rejections when pushing to the Kubernetes API server. Pruning results: skipping unrecognized cluster condition type \"PromQL\"\n \"ts\"=\"2026-04-10 12:04:14.748740\" \"level\"=0 \"msg\"=\"Starting identity container acquisition\" \"count\"=1 \"specID\"=\"Customer-should-upgrade-a-nodepool-from-4.20.z-to-4.20.zLatest|pid:893\"\n \"ts\"=\"2026-04-10 12:04:14.933427\" \"level\"=0 \"msg\"=\"Successfully assigned identity containers\" \"count\"=1 \"attempt\"=1 \"elapsed\"=\"0s\"\n STEP: creating resource group @ 04/10/26 12:04:14.933\n \"ts\"=\"2026-04-10 12:04:14.933646\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n STEP: creating cluster parameters at control plane version @ 04/10/26 12:04:19.473\n STEP: creating customer resources @ 04/10/26 12:04:19.579\n STEP: creating the HCP cluster with version 4.20.17 @ 04/10/26 12:05:04.201\n \"ts\"=\"2026-04-10 12:05:04.201160\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-jnm8s9\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/10/26 12:16:02.227\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/10/26 12:26:01.442\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/10/26 12:29:09.867\n STEP: capturing node release images before upgrade @ 04/10/26 12:29:10.041\n STEP: triggering nodepool upgrade to version 4.20.17 @ 04/10/26 12:29:10.08\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/10/26 12:29:52.504\n STEP: verifying node pool GET still reflects the new version @ 04/10/26 12:51:54.086\n \"ts\"=\"2026-04-10 12:51:54.286433\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-10 12:51:54.286512\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 12:51:54.379693\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-jnm8s9-rpjd2w\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-10 12:51:55.257214\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:55.257316\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n \"ts\"=\"2026-04-10 12:51:55.257391\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:55.259038\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\"\n \"ts\"=\"2026-04-10 12:51:55.259056\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 12:51:55.353380\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"pooledNonMsiScopedAssignments-jfl6npesxtst4\"\n \"ts\"=\"2026-04-10 12:51:55.415329\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-jnm8s9-rpjd2w\"\n \"ts\"=\"2026-04-10 12:51:55.578837\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-10 12:51:55.709848\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-129\" \"deployment\"=\"msi-deployment-129\"\n \"ts\"=\"2026-04-10 12:51:55.860303\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-10 12:51:55.860430\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-10 12:51:55.860601\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 13:02:14.746040\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\n \"ts\"=\"2026-04-10 13:02:14.746065\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-jnm8s9-tb54l5\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"1656f2b598b891e3f588923f365c70c85887da29099479a61fa374a63b758405","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"7ba7fc6de5bcafc9ea168179205860c649c276fe4162534ff6f785c88ed1a1bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"e115f0d91ca258d05c604dcc10d976dfb30b4ee3c98378e747983d1d19c90761","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"4b59ed1e7bc1c9174eb070f50a729e281f859e483dcfa086bc53ac0d3016c8bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"b052f95c7ac43877358d9b58edf1d1cf79d94d9b5351ee84ab0314fd489ecd4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"d24344800d201228d64db010e816a5e25cf74acb86cc4c5d05002723bf4f2156","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"e2072d0c469b46a1d7f7d3e238ea467288fa9ef089199ab06d9636c4c900cdef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"e5e871a9a2c633cc721fe9ca827b4448e9e818af940c191290f0b5f0ee7e369f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042523562169864192","occurred_at":"2026-04-10T08:42:20Z","signature_id":"977c5a79955b703d5d52fa873359cbadc299b6ed68168fe22a6de3d1a85024e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"1b3bdf86f9c812b2a7b8f5c8ab5b543a1b2a695fcf36e4c92a1990c7902ccee1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"774c8b9425c53dec4fa564c04d6b0b55f586c357fb5487ed255e12937b8edcaf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"c269ac0dc1e4722962bb2c16446c66884aff962620fc94e198f5797dd1547de2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"eb113763ca9eb39cbfbf3f6d9b2910a4d656864f89ff1075609a1e9aa9129dd1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"456956c00d2cd03fe0e688f32f7d08a0fab5648b265b6120efc3254d3963f16b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"711813ff0ef173ae4c74b1746c0d4883d500104597abcceb58cf5fd7e1fa6fb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"74a2f372f73f571cf44d910b50c9e1559c9dfb4de2568689b9ffb24d2fdb9485","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"7a3430cc1684e21012eb2689e5bed9d9e16bc291639be081252556bdf6682079","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"85b95c9895e4e0f7c2fd44a5c6864b35807d9f8a75a3b9bb1666804bfe472133","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"a09d0db5dedab2fb189aca9a3aad2190a32644a96b9529a61aa4af99f28d3beb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"a4e2d1c4ce37762825496b9afd8965783d6fd32cc144b275c851da1948e6b75e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"ae1f25418990f1a3fcdf9ce89f3756ee2927b16ebcbdfc62e13edc53b8bfa567","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"b7bb55f3055dea91e4e2e0a35768691dd8a3bb6770c7125349f05399ef297db8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"c79c0556df8ad3acfeb27115565aa651ad5885ac90240887372b6c08276ea8a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"cb1103453558f656c64bcf405755d3e4078107305cbaf3d067fbc78e283e95ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"ccf1c285155e7641c812ec1a64a480027fe0f5935acb52b42c0d556cedc15270","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"144724c9be25fac726ad599942218985e7cb1c767a58c2aaf40d3e1a56b3670d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"25112cf607019a413ef4df94318c39f9c02def0919498039896d48d73186ae50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"947d109d8d43c9f5ce0a3ad65337cbc76c5e5f47e6a6e8e25fe216f7aba3aba0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"9b8dfe83a7175b3c3ff1dfc846c5bcb0af067c5e163ffa5144f809ba04cb7fac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"a44703a6832a34a86e069e863ce95031cec6241d092adc5005108861b3a236fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"bcf9a78e2f0d6aacfd2a03648c0125ea09a1d71a284597b72aa79e4ee078e01f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"cf618007959110b43c9382ce0265e98bf5e4ae17099efc8020ed34e576a1d184","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"d26642690096c377827f89a1b690bbd6e91c6a40cad085c73f784a6bea3470e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"e7b16355619287ee49663b13c0880b88572d6800c4fb83e0f33d39be07eaf924","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"f4da3c4b9ddd4ac2311fdb82b1a39837f9b3df6336381691c7f2b8bed4807b49","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"0a2db9b0c728d07e9e80df888d3f0ca03fe60b12602e85ef9065fba88fc78ab8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"35df135bd95fbdb3da8af53544341ab557e41e5de58dcd707fcc3fd1aa7d8ba8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"389055155e0e9e664a2b8bbcab0e62ebe98717b50141fe5e2fc8bd747cb12977","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"905b4519fd2ae631d09bec2ad336fe05d492e64ad60a1c4840fb63249115793c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"0e3db37633accfbbb3e81a3af573a728d2faa0040f7694a8ebd9aa388f5fbf5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"0fb0bab9c886e219989c17756195bd6d490d3111d3d7e44bba6ea3fa77bb66c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"809d7844f1a48c70c3423a63cecd4101fb58ab0d6a342e98acba6ff46c3c4077","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"b5735b60f9dfe41ca07bdf25a9ce5b92363a013abfc259e91caa0c98738482cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"f2738bf06536a42aa77a9502fb7deb243a43c8d2c689ef96192d23fe73aa8ea4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"342fd958a98f57fdbc69171f73b9979d6370fb082d12515c8e5436854e410338","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"4f4470447da4e0eec3b1263376a6f4e4eabd73562a0c2787841c7a1fb3de6fef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"bc16479542f5518f199cdf90d49056dbd514a7388517335dffc90cafedff085b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"ccbd3aa0996ca39ea8f78950f49effcec35d7962ca56c70dc03209ef9115be87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","signature_id":"500d409f3d8fb29df0602cb09997135f46bcf75fd9f7fb14e42d1ce74f68449c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","signature_id":"80425a41a776bc9c054fe74e89b94fff4fb1f63049b7a42543353153422bbf16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","signature_id":"771a9b202a2b5fcd9bc2f6284c90ce9ab0922fc86ee71395d36f43a8a781d0d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"0eb17ac0fa73cbaf6bfd96b147f1e87899b4eb9122843f96122b57a919ecf965","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"49ed41086fe7a9ec2b601ce88c197476ebb461138a74a4df1c55c826a052fa0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"545f07c5de94eb876fe5c60d1fd48aabe80ce9fe76291be55b34245f3cddfa0e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"84b0d7cc081dfd6750b672d63540a59e19f9f30758d835bf5f5ef823dbac3aa5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"a7a3a791a16cdc30955425371d11ec180f558068ee2777f1b760ec0eec8c71db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"ea06de1114c3d8f9ae2bd132acedc5c7fdc738b85c2b59230a4f78070b8f7429","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"f1e615836088d3a70d52b7c6f718a74bd4b874c9412a0b8713b7bb1e5df7bace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"b7d39ccaa33471c6fe6d2423539073913dae6879ccbe030526e8f6b3ab342f0b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308037523705856","occurred_at":"2026-04-07T00:12:16Z","signature_id":"ab4eb11ba454086e3a61bca639145624e40a3a0c90bd62c730548e8ec4d56dc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041215587668987904","occurred_at":"2026-04-06T18:04:54Z","signature_id":"5e3ed17db3d12f1ab35835192b8926fafab58511a775c141ecb3fa76b36efd69","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-b5cpds\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-b5cpds, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":58,"runs_affected":10,"failed_runs":10,"run_impact_percent":24.390243902439025,"anchor_occurrences":58,"after_last_push_count":124,"trend_counts":[21,1,10,0,22,18,0,7],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-16","2026-03-23","2026-03-30"],"prior_runs_affected":23,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e54480\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-9rklxc, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-9rklxc in resource group rg-cp-ystream-upgrade-4-21-rqz9bx, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-9rklxc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-rqz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-9rklxc in resource group rg-cp-ystream-upgrade-4-21-rqz9bx, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:126]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000e54480\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-9rklxc, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-9rklxc in resource group rg-cp-ystream-upgrade-4-21-rqz9bx, error: failed waiting for cluster=\"cp-ystream-upgrade-4-21-9rklxc\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-rqz9bx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster cp-ystream-upgrade-4-21-9rklxc in resource group rg-cp-ystream-upgrade-4-21-rqz9bx, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00150c090\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-c9kcwx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-c9kcwx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-c9kcwx, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:90]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00150c090\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-c9kcwx, error: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-c9kcwx\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster arm64-vm-hcp-cluster in resource group arm64-vm-cluster-c9kcwx, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001727110\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-nftcd7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nftcd7, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:148]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc001727110\u003e: \n failed waiting for cluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-nftcd7\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nftcd7, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043449495512944640","occurred_at":"2026-04-12T22:01:39Z","signature_id":"f085e719c345a23826574abbc68acc96bbf8b703267d8308f53284c6e50935e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043449495512944640","occurred_at":"2026-04-12T22:01:39Z","signature_id":"f71e8c376fc5b8501f08898b7575823cfe5e3825557dc723e034e89a29c77ce0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235269712285696","occurred_at":"2026-04-12T07:50:24Z","signature_id":"b7f633a557efe37d24363e9da2101813d96fc59ed3e67442b699109106b1da1b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"26c882ed9f5935d25335e778b272459ba40a15dc6440bbfc244a4a245fd17591","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"a90102ab9f84d2dfa7398c630b29d8fc08bc5b5ef3dd59dd452ea3cc95edaad8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"dcf6acef2dbe6d4d655bd9bb7ee305bd1006ae34fce3651cd024171822d619fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"dfbe816c5bb5729bb4488f044f57146c9b1b6075a693f271404a68f7ef6e6d42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"4417f9ebee113ede569a63481c296343013ef4c0dd63cb13ca927a1c9d0575a6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"f4a2a227d30c0371e3cd0edc206bb57bb934f7573ea663644af9bd1f992bdc4f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"f9d8e75397a206a6268f651b0be6637d057a861d0bbf464e067b67c82a1421dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"0d0065a85884f57dc55146b3cd6f9e7f652f0498eab4c9f7cd357e374c69da98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"3342dc667feb6683fa8d37b7adb8bf7cebbf78de37e827acc0e3826580e28e1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"3acbfe85d42d4b2af0028ce47dd46b2ecf6328bee6e74b263673d44b4664b09c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"3c74c2b7b0d98970008636e4702e3dafe55cd39f4902e823fddd1b1caeaf1437","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"466a1bc425e7596e811a1a5b201d5b80622e0baf5fe5f127acd6d33a30be9d6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"7c5da14974f842e2f7863d15e9875fdc09997be97e160621aa4a3cd853a7b438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"a8c8fd1a5a8ee69127b99cd5468e4422b9f95af227abcd6b8f9e4da53d12eddf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"0591505a8d0d09ee27d773b578770c0879cc0fd3c1615c87b0b878047697e942","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"633be0bfad8cef28de804331d1e9aee9b444f7683897f6917a5919e6188f7c6c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"685faef7c56cc1e714facd7df48e1f0acf7fc3632e9fd34b791af4ada2983c56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"79e17694bc7eb6852bb1fb17ca10ba1644573ee030a39c7db08020c8c4393908","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"d4beb9840975059b78ebb2bbdc1f3bd005e206d72191c83051709934b2acc299","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"ef4ac3a0886e7bfd19c86141172c407d907b090c56c349762c5baab5d47ea58a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"a1ffbe285ac5025d1bccac53237d64f36796874c70d5c5ae5d58b795494658ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"ee3c35e66876b8446d35f42a776892654f3b6ea66fb1b4ab7ae0775c93d7a8c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"1434b8b1beae1e2462f904ed8c7fd391d5c039e029a420e6cf6d081b0c0d0b0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"1ff88c7cdd3499f0b611863b832cf641a1174675f55466b3a9b7339a91830ff1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"20d80203cbad1be4b26371494dcc6cc544922c19a12c074a01d8bb2de59736ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"2ee08db4e3b091585cb0d893f015ff959a35d3c08ed6f9c2c96d9d598749655e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"330cb64c3c554ccfae42debfd0d76fd8797f0c0fcaa880bdd959f65399919c88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"4b476ec19b339bba387ec9743367a687655cf83e55ffc3d32165311f768789cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"4bd17e5262806660ea3cb7307dacae443f38c55f69a35963fbf5b93d093d7077","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"5143f678069366b9c72b55d545d3d5ed863e67f5752c369766a1f2194e58b299","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"51d263ce91b4409e4460d92f782dcbcf48a02d5d56f79d54b7af9e35ec33c3be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"66f5e30a0278bda3bbd019ac2f4b9bb0e570da30f3b70999fa97794a1ccf19be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"763cc5ea46d3724878f97a7c2aeb317ebbf14cfa39e2f628b3b2132b6b57ae00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"9fa9e212bfc5e515204db2c8cda1c2e9df00f18528747dd58502103dd58209f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"a3601dc891786f1e347882a854cda79072206de659df846624d91488873d6788","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"ac3b4a72760a724c1b769b45f8e59a39b612336b94a6c45e7386811540975018","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"b8be7be9b69c391c1558c0d3fba34bce9063369ea28ca2c53d29ce2a33a4fdfc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"ceabc22814b9276f1ff45e9ae3bafd075ad167466b20198195bf855b7b2d2c52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"d7722e9b5999c253e78f65c34ee8909e1f5f33ad52d970eaea64c9e567cf6c35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"d77696467a2d442aa450fda4be7fa109a0efae503aae9c34c88fb70b01c15bb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"e02eea0acb0857847efd731148c95427f368bed721e2406ad6b6e8de41fad190","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"e1c9b11aabb04b01cf379a5ddb36533a49ce1283f8171a5e595ace8671ce7942","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"e777e665a1ca73ae9f55c94dd841945efb9238d4af249f7d2ae35961c1bdc2c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"f04a5eed1e2f72f7a0d6b6ed6f9ad8509a8864946852b1905eb5300b43ce6b51","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"19b8b89f21ef63dfdd459a5ae0dde3ec9e67b8586ab0532fdd1a4d72e5dd9677","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"3298511640d7e93d00a720ae5553b54712470ec9b619dda50523cf8dd9b743df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"45549f3d568ceae199450b06a0f29ca707fdcd572e216b08b615100bc902a118","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"48d66cb903bed78ecc5b3870134f6236fde8da937c87322b0e8aea6adb477cb3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"490a38b6630e9d08af57dc89b154f250a304afaf6ce6fbdf3395ae1e55a94afd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"4e32fb5d7ee25f5f0a0bef0524e92b9066de0d63ef1fec9063e4350475d0eb88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"513321780d22667c85d39965dcd485e1739ec694dc7dfb6d2311d111c6478061","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"5a84ec606cf94e594e22c62e9fe85fd628da238028d88ecf1ca775367060da89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"bc0af7ca1eb571869bcfd45d0933b754212864c3ca865d87bc33745cfdc01aa5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"cf73ef57a2796840467d871dd1126493528568bf6b577c2cbd568c2f109e27c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041274391769124864","occurred_at":"2026-04-06T21:58:34Z","signature_id":"b6a06340d62d98c098203a1aba2a3caa8ba91ba12899e2bd27a359543dee5d54","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wmv8g\" in resourcegroup=\"admin-credential-lifecycle-test-46ctqx\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wmv8g in resource group admin-credential-lifecycle-test-46ctqx, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":41,"runs_affected":10,"failed_runs":10,"run_impact_percent":24.390243902439025,"anchor_occurrences":41,"after_last_push_count":82,"trend_counts":[0,0,14,0,4,15,0,8],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-03-30"],"prior_runs_affected":19,"prior_last_seen_at":"2026-04-03T18:45:33Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":6},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:127]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00109e3c0\u003e: \n failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-9r68hs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster negative-tests-cluster in resource group rg-negative-tests-9r68hs, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:127]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00109e3c0\u003e: \n failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-9r68hs\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster negative-tests-cluster in resource group rg-negative-tests-9r68hs, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00158c090\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-l2hzs5\" in resourcegroup=\"admin-credential-lifecycle-test-lsrgg4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-l2hzs5 in resource group admin-credential-lifecycle-test-lsrgg4, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00158c090\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-l2hzs5\" in resourcegroup=\"admin-credential-lifecycle-test-lsrgg4\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-l2hzs5 in resource group admin-credential-lifecycle-test-lsrgg4, error: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00170fdd0\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qwvv9r\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-qwvv9r, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00170fdd0\u003e: \n failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qwvv9r\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-qwvv9r, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"34600865e38ac35a541d79aaa4d5da3563a95440d24d43e67db2debf2680d57d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"539d2854384b04e924a353ceda685297e15af2f833a40564d0b65b3c6db44c9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"684e1a48663f18039c6e9fb3d0f3f6c3abef457f92dc92f283710d74801b8817","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"b44a31f79ac10196efc91d3ef7ae148aac7089bd62f746cab69d0c7474c0b049","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"b51c0415c0142f65afdf441528b88f526ca714ec7e530d28ee2a0e8efd9ea1c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"e98292989102ada55444807f652dc01eedbb9a8cde699b434ae38f384786118a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"eaa51860bd4eee567ac1cc72fa6c8c796677f8a9d6325b556371d081de0d13f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"ed652b47de65d3fdeec913740fbfd5215dc1f19eb45a5a7eecbe183b4b5d7fd5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"2e4c68add548aaa6737eadeda2aefb454957e84c997a068f440595794fe5ea8b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"324ffdaba18d3147faf9a4285ad6fa60df88e7720fcd0b876adf23da5ffa8578","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"ad1a784b6233c088bf1cbcdab05052ce4206e83faaf04a30cff15ea5dc427b03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042684774031560704","occurred_at":"2026-04-10T19:22:53Z","signature_id":"a9d92a762e487ff6e17f5d7d33bf21ab33f872b713827d1508be3bd3f1e9a462","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"094a6c2314265f60b76e8fcbe84a2c8e44259e101a52c5a4c142e5713aec8804","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"5a88ff7c5b00148eaccf0a7f4dea26b63c8ec41eaceae91a95ba6b5e42525ddc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"7a0ad282eff611158e8e43418bc514839cbe0023046786a1ce9aab3f8e0a2e7a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"97ed143ef45d8dc14377b681c99ace8f580e81f564813b645e8377e8d924dee7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"9c9530d101e12ab5a650c8fc6e9715606e353ecc95c0653e2075234a94e464fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647665363128320","occurred_at":"2026-04-10T16:55:28Z","signature_id":"e1e9672e98e7c5f28b1706f8fa646f2740d0999ee129fbdcef9034e7e05cce00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"a87063a43d794d60b0e265445eae9710463acd30489634d881a98f9be587a1a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"e72458e45371568f1114245b29ad35b4f5307ce068bdec27027c68de7680c7db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"5012567a501f3d37567b8d2827f0fd6d46f0b92140488b498409244711ec15f7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"ba350b37506f60e30bb4bb27da0434f337a2464ab0c305b5ebf8b0a627f56e6c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"de4a5ce3ac3c5e9aea9290d16182f031cb6e3c37df7abdcce982c39a9f3fc408","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"a0eb876f1dcc8d31439a1e9ac1c75b37fa3dc26ff9193f4bfa7d679cba33c237","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"01edd05054d3d9450ab30aa909069d2d80dc296ce304743832a27fede9a35a05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"7843d873e84cfde4df6aa09b0a6bbf5526a5df8bc96e125218f65bb90b5c80b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"95ffaf23bec65ec54f5a233ebbd9ba6ee14445df48b2a02c2a9c8c45aa82d42e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"1afd98ace371047c0b6f30472486f90f36627888de8ef560568f39a8a9f7eefe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"34c8538cd1cf99db24364475fbc228e23e26d2362e2a83d8f0fdfb52ba4cf1f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"38826b533269d7a9bb238361448ad259ba2286646e96c6502152a76dcb1076fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"5e6c3c5c47b005967ec25cd98cba3a5126b257a4788d48b77e5a129547e95eac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"6d1fba2228413ee88801c71b444cc9f8d5620d8f429f71bb349c575fb7b5f3b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"db21d7b4d2180dfb67585d94a9c18b9307efdc3021536e3fe62d1bfcb1343968","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"e3bf0e48b6a7ad4445c218645ad83a075e948fb4e051c71c17cf9d0ae2652300","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"203e2c7ffff61cb7e8a9241ca3de6134343765d2bd071522facb4429605fa8c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"23cb7cad2d50cb707def47ef79150e901e0494d1b2b7151fbbaefbb3e6bea082","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"5be661969e52e9a052ab0134a9f3bdb6e4e82b3b6b5d2c04656940f3faa90b53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"697cb4e2d08f75c3268c0a07e907dadf1330b74f8378572502578afac4409d4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"8a59fc555efc0c6395952c408b943ce13eb2659f4641992e1435b4f7478938ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"becaebe81abc8fbb81aaadb376c5d1d5b254498b9ea945ec6a9e215bb1ffaa52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"fce7f9cb30c215532540321a2400b35a1f573a9281d5e92cf3123f1c7f7dacda","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":24,"runs_affected":15,"failed_runs":15,"run_impact_percent":36.58536585365854,"anchor_occurrences":24,"after_last_push_count":61,"trend_counts":[9,7,9,3,4,0,0,1],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-09","2026-03-16","2026-03-30"],"prior_runs_affected":38,"prior_last_seen_at":"2026-04-05T18:04:03Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":7},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:35]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:44]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041908135588794368","occurred_at":"2026-04-08T15:56:50Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041908135588794368","occurred_at":"2026-04-08T15:56:50Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041865685600047104","occurred_at":"2026-04-08T13:08:10Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041279040924422144","occurred_at":"2026-04-06T22:17:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041274391769124864","occurred_at":"2026-04-06T21:58:34Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041271574195081216","occurred_at":"2026-04-06T21:47:22Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041215587668987904","occurred_at":"2026-04-06T18:04:54Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041074948453175296","occurred_at":"2026-04-06T08:46:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041074948453175296","occurred_at":"2026-04-06T08:46:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041074948453175296","occurred_at":"2026-04-06T08:46:03Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":19,"runs_affected":9,"failed_runs":9,"run_impact_percent":21.951219512195124,"anchor_occurrences":19,"after_last_push_count":36,"trend_counts":[3,1,1,0,5,8,0,4],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-30"],"prior_runs_affected":16,"prior_last_seen_at":"2026-04-05T20:06:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e9c280\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rtxghs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-rp69dr/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000e9c280\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-rtxghs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-rp69dr/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001380380\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-pb6kcg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001380380\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-name-pb6kcg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-name-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d2c120\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-xgtzgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:185]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000d2c120\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-xgtzgf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235269712285696","occurred_at":"2026-04-12T07:50:24Z","signature_id":"23e5caf0423c56c52bfb51f206d3afc2f2069b3270ea02f2b3f83a42b6e8f24b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235269712285696","occurred_at":"2026-04-12T07:50:24Z","signature_id":"2d0245b34511d35876518838701a481d1d672ddddebc1289af52ac81995255b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235269712285696","occurred_at":"2026-04-12T07:50:24Z","signature_id":"b09aae02dfd8bb593ea38bd60cdb43d4670d74da122becd1a7818ff311d7db12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"4231e38cb38296682e16d97c0825fa632e1e0f5259180d26ca3e3bad22b10808","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042701416304218112","occurred_at":"2026-04-10T20:29:03Z","signature_id":"55ef6ef4d9ea61f11d4d23769864db3f0d507685dc0ef554cb5890b869bcca8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"1b2c3288fde7d0baf2d757029c5c1cfe45b7c15828e1e4ac53ffd4d897080b18","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"3f552bc1f85dd1b4bd1189c16824c26c1b464f002c113d93d56f72f56c0dcc27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"e09f5da2799329b5ee78a500dfa49b480e4ee88c92315c4c097a19512e99ec74","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042647552989335552","occurred_at":"2026-04-10T16:55:01Z","signature_id":"f726f5cdc0a97832459d0a83f77b5fc8d60caabf1f5995ed8b497222589a4f3d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"8b1222a8560e2917674611da65a4169d2346b2401f470a0385b6c34c7da27c8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"3d519ffa44179193aedca896e5792ca8b7df0696ac0a7f979d1d282a5702f1c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"f2fc8aa99648d2d62ceefe9d3e42b8eb17da274af35f704f60c14515a7ecceb4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"288c8d145446a41254ecea6baff1c303123323830880407686bcc2a82988756f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"68b4f79c4388b2a38f2fc7f55a3630604102f569c8dd5b906c95a4901a1ac61c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"97312cc094263b3ba5463d178634f674da6726b6e062cecc0738c80b9e6d6fbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"aa2f8fa3e1bb2876e9c5056d271c2441da23870796e6387dd2d2709f791ed027","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"c44640292de911bca3ae3026f422b2d682df399df9a22d6e4536f3e7e6da6324","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","signature_id":"46a267eec014b8e48c35e8603c9ead475405836ebd27b349d3b51b246bd00195","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041074948453175296","occurred_at":"2026-04-06T08:46:03Z","signature_id":"6fd38f04ffcbf33044f9345daba9d5b99aae6f3ccf9b4ee5177e79aa64761c40","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":14,"runs_affected":14,"failed_runs":14,"run_impact_percent":34.146341463414636,"anchor_occurrences":14,"after_last_push_count":21,"trend_counts":[1,1,5,0,3,5,0,0],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-23","2026-03-30"],"prior_runs_affected":21,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":14}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T13:03:33Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T13:01:55Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-10T10:44:38Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042557690818859008","occurred_at":"2026-04-10T10:57:56Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042523562169864192","occurred_at":"2026-04-10T08:42:20Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042522288930164736","occurred_at":"2026-04-10T08:37:16Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042387582653829120","occurred_at":"2026-04-09T23:42:00Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366836473237504","occurred_at":"2026-04-07T04:05:55Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366521514561536","occurred_at":"2026-04-07T04:04:40Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308229962567680","occurred_at":"2026-04-07T00:13:02Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041308037523705856","occurred_at":"2026-04-07T00:12:16Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041215587668987904","occurred_at":"2026-04-06T18:04:54Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e8661d212ff004d96b95068ad8af2191b7abda7079211b806e672edffcea0fe","failure_pattern":"CreateNodePoolFromParam timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":7,"runs_affected":3,"failed_runs":3,"run_impact_percent":7.317073170731708,"anchor_occurrences":7,"after_last_push_count":11,"trend_counts":[0,0,6,0,0,0,0,1],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-16"],"prior_runs_affected":6,"prior_last_seen_at":"2026-03-19T06:26:31Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00130e390\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-czfgf5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-czfgf5\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc00130e390\u003e: \n failed to create NodePool nodepool-128GiB, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool nodepool-128GiB in resource group clusternp128-czfgf5, error: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-czfgf5\" to finish creating: context deadline exceeded\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000958690\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-pmp787\" in resourcegroup=\"rg-np-version-upgrade-pmp787-kwgkwl\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001248460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl\",\n },\n \u003c*fmt.wrapError | 0xc001104340\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:154]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000958690\u003e: \n failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-pmp787\" in resourcegroup=\"rg-np-version-upgrade-pmp787-kwgkwl\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool npupgrade-4-21, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl, error: failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001248460\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool npupgrade-4-21 in resource group rg-np-version-upgrade-pmp787-kwgkwl\",\n },\n \u003c*fmt.wrapError | 0xc001104340\u003e{\n msg: \"failed waiting for nodepool=\\\"npupgrade-4-21\\\" for cluster \\\"np-version-upgrade-cluster-pmp787\\\" in resourcegroup=\\\"rg-np-version-upgrade-pmp787-kwgkwl\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012d86f0\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001247440\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h\",\n },\n \u003c*fmt.wrapError | 0xc0005187e0\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012d86f0\u003e: \n failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-b8kv4h\" to finish creating: context deadline exceeded\n {\n msg: \"failed to create NodePool arm64-vm-np-1, caused by: timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h, error: failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001247440\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateNodePoolFromParam for node pool arm64-vm-np-1 in resource group arm64-vm-cluster-b8kv4h\",\n },\n \u003c*fmt.wrapError | 0xc0005187e0\u003e{\n msg: \"failed waiting for nodepool=\\\"arm64-vm-np-1\\\" for cluster \\\"arm64-vm-hcp-cluster\\\" in resourcegroup=\\\"arm64-vm-cluster-b8kv4h\\\" to finish creating: context deadline exceeded\",\n err: \u003ccontext.deadlineExceededError\u003e{},\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043235187642339328","occurred_at":"2026-04-12T07:50:04Z","signature_id":"dab7a47a660e304b90a9c8500d3d50c0460ca30e6fdf5697f46adf1a50eb3120","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041366674841538560","occurred_at":"2026-04-07T04:05:16Z","signature_id":"f15184542c39bb774c3be8e478ae04eeb6b9843c71ff6dda60d4522220a27a0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"325ed533ecfab9ad14356e1c3682e386e21134251bde23aff995a4190cec9728","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"38bb290cdcd5fb0fd6fc57972bfe6e55acd2c90b588687aa9b866e4af14f4e0f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"4ff69f402a35f8589728a46de86bc483f764602ca78dd11c5bdc70a78155a06e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"58723eeca32480c2a7147989b14ad5770f17d9613a970f9c8d123ee9128db707","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041310807207186432","occurred_at":"2026-04-07T00:23:16Z","signature_id":"7ed989922bc03f5a77baa84831fdc451a05ac58926599595e31930fd3a9d61de","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"0839b6ab49894fcd8abe58a0a618a81c908f5565776270e1693153b13a4d0d9a","failure_pattern":"tls: failed to verify certificate: x509: certificate is valid for \u003chcp-api-host\u003e, \u003chcp-api-host\u003e, not \u003chcp-api-host\u003e","search_query":"tls: failed to verify certificate: x509: certificate is valid for api.ea-cluster.kv02.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.ea-cluster.50y5.uksouth.aroapp-hcp.io","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":4.878048780487805,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,2,0,0,1],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":3,"prior_last_seen_at":"2026-03-26T02:06:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:107]: Unexpected error:\n \u003c*errors.joinError | 0xc000b2f950\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.ea-cluster.50y5.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.ea-cluster.kv02.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.ea-cluster.50y5.uksouth.aroapp-hcp.io\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ea-cluster.50y5.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.ea-cluster.kv02.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.ea-cluster.50y5.uksouth.aroapp-hcp.io\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:107]: Unexpected error:\n \u003c*errors.joinError | 0xc000b2f950\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.ea-cluster.50y5.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.ea-cluster.kv02.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.ea-cluster.50y5.uksouth.aroapp-hcp.io\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ea-cluster.50y5.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.ea-cluster.kv02.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.ea-cluster.50y5.uksouth.aroapp-hcp.io\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:168]: Unexpected error:\n \u003c*url.Error | 0xc001248ab0\u003e: \n Get \"https://api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.q6i1z1a9v3j3w8r.qgw8.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:168]: Unexpected error:\n \u003c*url.Error | 0xc001248ab0\u003e: \n Get \"https://api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io:443/apis/config.openshift.io/v1/clusterversions/version\": tls: failed to verify certificate: x509: certificate is valid for api.q6i1z1a9v3j3w8r.qgw8.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.q6i1z1a9v3j3w8r.lr4g.uksouth.aroapp-hcp.io\n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n \u003c*errors.joinError | 0xc0013be210\u003e: \n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyBasicAccess failed: failed to list services: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:108]: Unexpected error:\n \u003c*errors.joinError | 0xc0013be210\u003e: \n VerifyImageRegistryDisabled failed: wrong type of error: *url.Error, Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/openshift-image-registry/services/image-registry\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n VerifyBasicAccess failed: failed to list services: Get \"https://api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io:443/api/v1/namespaces/default/services\": tls: failed to verify certificate: x509: certificate is valid for api.j5b3t8t5w8i3j5x.8h0z.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.j5b3t8t5w8i3j5x.83aa.uksouth.aroapp-hcp.io\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2043449495512944640","occurred_at":"2026-04-12T22:01:39Z","signature_id":"7d220b2543544fab9d8aa7473d9c69855bc87d112af01810d36ac6af3128dc5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"2769ee201200e30cdfaa3da531246e1754a9bebacbf8e5667de0496fbc64cba4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042063253525237760","occurred_at":"2026-04-09T02:13:13Z","signature_id":"691bb7eb88df394a0a911b08820aeb36de06814b9281babcaddd7c570f135e34","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1e9c6c185c09aa6c23477c53c1d3d7fb1bb60e041c3cb718558ce145b6718d4d","failure_pattern":"Cluster should become ready within 45 minutes","search_query":"Cluster should become ready within 45 minutes","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.4390243902439024,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[1,0,0,0,0,1,0,0],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-16","2026-03-30"],"prior_runs_affected":4,"prior_last_seen_at":"2026-04-05T15:03:46Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Timed out after 2700.000s.\nCluster should become ready within 45 minutes\nExpected\n \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042423946946547712","occurred_at":"2026-04-10T02:06:29Z","signature_id":"8a441c3cc9a1724fd565ecc68f397cdfdb078c056f41b0ab5661bd4e719eb328","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"21d4ef5ea099bcab4c9b17a22b87833125050f5319c2fc11c95f48e6b41e5281","failure_pattern":"ERROR CODE: Conflict; detail message Cannot update resource while resource is \"accepted\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: Conflict","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.4390243902439024,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":2,"prior_last_seen_at":"2026-03-25T10:46:15Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0001769a0\u003e: \n failed to create HCP cluster list-test-cluster-p4ct5p: failed starting cluster creation \"list-test-cluster-p4ct5p\" in resourcegroup=\"cluster-listing-m5pqxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster list-test-cluster-p4ct5p: failed starting cluster creation \\\"list-test-cluster-p4ct5p\\\" in resourcegroup=\\\"cluster-listing-m5pqxl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000176980\u003e{\n msg: \"failed starting cluster creation \\\"list-test-cluster-p4ct5p\\\" in resourcegroup=\\\"cluster-listing-m5pqxl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000c6f590\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Date\": [\n \"Mon, 06 Apr 2026 18:28:48 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/f8dcff39-92dd-4b53-bfd5-a54e320e9c99\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260406T182849Z:c7e70c50-1876-4215-b20d-f5fe6a8b0f35\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 6C605D2072634CEB987C0EC2A7E6913D Ref B: CH1AA2020603011 Ref C: 2026-04-06T18:28:45Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"057ea031-c736-4cfd-82cb-048ffa7ce574\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"c7e70c50-18...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_list_no_infra.go:85]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0001769a0\u003e: \n failed to create HCP cluster list-test-cluster-p4ct5p: failed starting cluster creation \"list-test-cluster-p4ct5p\" in resourcegroup=\"cluster-listing-m5pqxl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\n --------------------------------------------------------------------------------\n RESPONSE 409: 409 Conflict\n ERROR CODE: Conflict\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"Conflict\",\n \"message\": \"Cannot update resource while resource is \\\"accepted\\\"\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster list-test-cluster-p4ct5p: failed starting cluster creation \\\"list-test-cluster-p4ct5p\\\" in resourcegroup=\\\"cluster-listing-m5pqxl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000176980\u003e{\n msg: \"failed starting cluster creation \\\"list-test-cluster-p4ct5p\\\" in resourcegroup=\\\"cluster-listing-m5pqxl\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\n--------------------------------------------------------------------------------\\nRESPONSE 409: 409 Conflict\\nERROR CODE: Conflict\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"Cannot update resource while resource is \\\\\\\"accepted\\\\\\\"\\\",\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cluster-listing-m5pqxl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/list-test-cluster-p4ct5p\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000c6f590\u003e{\n ErrorCode: \"Conflict\",\n StatusCode: 409,\n RawResponse: {\n Status: \"409 Conflict\",\n StatusCode: 409,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Date\": [\n \"Mon, 06 Apr 2026 18:28:48 GMT\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/uksouth/f8dcff39-92dd-4b53-bfd5-a54e320e9c99\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"UKSOUTH:20260406T182849Z:c7e70c50-1876-4215-b20d-f5fe6a8b0f35\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 6C605D2072634CEB987C0EC2A7E6913D Ref B: CH1AA2020603011 Ref C: 2026-04-06T18:28:45Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"X-Ms-Error-Code\": [\"Conflict\"],\n \"X-Ms-Request-Id\": [\n \"057ea031-c736-4cfd-82cb-048ffa7ce574\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"c7e70c50-18...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2041215587668987904","occurred_at":"2026-04-06T18:04:54Z","signature_id":"5421349c67ce3eae3e20d07701ef2624c120001dceeca766d2af1759bc03dd86","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"4acfe441005f8003e7187d8bb50b5a3c1645ce7924b7f2de03ef6bdb2cb05e18","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Cluster '\u003cid\u003e' has an external authentication pending deletion; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.4390243902439024,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,1,0,0],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:294]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001508c80\u003e: \n failed creating external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-n5j64n\" for cluster=\"cidr-connectivity-test\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n5j64n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/externalAuths/ext-auth-cidr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Cluster '2pj8n2ov52hfcbj1lfepdkk77i8o48uj' has an external authentication pending deletion\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:294]: Unexpected error:\n \u003c*fmt.wrapError | 0xc001508c80\u003e: \n failed creating external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-n5j64n\" for cluster=\"cidr-connectivity-test\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-n5j64n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test/externalAuths/ext-auth-cidr\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Cluster '2pj8n2ov52hfcbj1lfepdkk77i8o48uj' has an external authentication pending deletion\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042558518153711616","occurred_at":"2026-04-10T10:59:58Z","signature_id":"71b0ec01edd9207bb394cba62c96fa66ee1e1f320a74672b8a730144deb83883","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"683173d3a6520478fe76027389479002ac2d2e1044c6fdc87add183d29683bd7","failure_pattern":"ERROR CODE: LockContention; detail message Failed to acquire lock: timed out; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LockContention","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.4390243902439024,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:141]: Unexpected error:\n \u003c*exported.ResponseError | 0xc001404990\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-rclvn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:141]: Unexpected error:\n \u003c*exported.ResponseError | 0xc001404990\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-rclvn7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 503: 503 Service Unavailable\n ERROR CODE: LockContention\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"LockContention\",\n \"message\": \"Failed to acquire lock: timed out\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042191604369854464","occurred_at":"2026-04-09T10:43:15Z","signature_id":"87ddbde6dce1408bf8bb9c3fe0aef6b8db9623c90da591e311e5dcaaae1ec7c3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"635a2afde9e3afd24ee6abfeb373ae222c4b51e9e9a5fed30800c3f45ff07db5","failure_pattern":"get deployment router-default -n openshift-ingress: Get \"\u003curl\u003e","search_query":"get deployment router-default -n openshift-ingress: Get \"https://api","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.4390243902439024,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,1,0,0,0],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:180]: Timed out after 600.000s.\nrouter-default deployment logs should be fetchable\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d582a0\u003e: \n get deployment router-default -n openshift-ingress: Get \"https://api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io:443/apis/apps/v1/namespaces/openshift-ingress/deployments/router-default\": tls: failed to verify certificate: x509: certificate is valid for api.o6n1s5y5u2a8q7c.wjok.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:180]: Timed out after 600.000s.\nrouter-default deployment logs should be fetchable\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d582a0\u003e: \n get deployment router-default -n openshift-ingress: Get \"https://api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io:443/apis/apps/v1/namespaces/openshift-ingress/deployments/router-default\": tls: failed to verify certificate: x509: certificate is valid for api.o6n1s5y5u2a8q7c.wjok.uksouth.aroapp-hcp.io, reserved.aroapp-hcp.io, not api.o6n1s5y5u2a8q7c.i7sr.uksouth.aroapp-hcp.io\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2042060062305816576","occurred_at":"2026-04-09T02:00:33Z","signature_id":"039071a1f2cdd978847b2c39b4ab9740ad502f6c020afd94bf765b938f2cfe82","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa2262f041e66eb099e37b23695a2256e26370a4e886b5e34fb2907105c756d4","failure_pattern":"pod \"\u003cpod\u003e\" is forbidden: error fetching namespace \"\u003cnamespace\u003e\": unable to find annotation openshift.io/sa.scc.uid-range","search_query":"pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.4390243902439024,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,1,0,0,0,0,0,0],"trend_range":"2026-04-05..2026-04-12 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:344]: Unexpected error:\n \u003c*errors.StatusError | 0xc0012b7ae0\u003e: \n pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range\n {\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods \\\"wi-test-pod\\\" is forbidden: error fetching namespace \\\"e2e-oidc-wi\\\": unable to find annotation openshift.io/sa.scc.uid-range\",\n Reason: \"Forbidden\",\n Details: {Name: \"wi-test-pod\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:344]: Unexpected error:\n \u003c*errors.StatusError | 0xc0012b7ae0\u003e: \n pods \"wi-test-pod\" is forbidden: error fetching namespace \"e2e-oidc-wi\": unable to find annotation openshift.io/sa.scc.uid-range\n {\n ErrStatus: {\n TypeMeta: {Kind: \"\", APIVersion: \"\"},\n ListMeta: {\n SelfLink: \"\",\n ResourceVersion: \"\",\n Continue: \"\",\n RemainingItemCount: nil,\n },\n Status: \"Failure\",\n Message: \"pods \\\"wi-test-pod\\\" is forbidden: error fetching namespace \\\"e2e-oidc-wi\\\": unable to find annotation openshift.io/sa.scc.uid-range\",\n Reason: \"Forbidden\",\n Details: {Name: \"wi-test-pod\", Group: \"\", Kind: \"pods\", UID: \"\", Causes: nil, RetryAfterSeconds: 0},\n Code: 403,\n },\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040950439863652352","occurred_at":"2026-04-06T00:31:18Z","signature_id":"c525179ba8c84cfd1edbe946a68a62380981593d54d12a25e846f243622ca4b1","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}