{"meta":{"start_date":"2026-09-30","end_date":"2026-09-30","start_at":"2026-09-30T00:00:00Z","end_at":"2026-10-01T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T11:07:14Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":5,"failed_runs":5,"raw_occurrences":21,"matched_occurrences":21,"runs_affected":5},"rows":[{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":10,"runs_affected":3,"failed_runs":3,"run_impact_percent":60,"anchor_occurrences":10,"after_last_push_count":31,"trend_counts":[0,0,17,1,2,1,10],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":14,"prior_last_seen_at":"2026-09-27T02:00:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c2500\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-xhfvnf\" in resourcegroup=\"rg-np-noedge-xhfvnf-d6cdpdqmxzz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"name\": \"79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:42.49451817Z\",\n      \"endTime\": \"2026-09-30T17:39:43.993225307Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c2500\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-xhfvnf\" in resourcegroup=\"rg-np-noedge-xhfvnf-d6cdpdqmxzz5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"name\": \"79ccdc55-d7b6-4981-9ed0-35fdb60bc186\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:20:42.49451817Z\",\n      \"endTime\": \"2026-09-30T17:39:43.993225307Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-xhfvnf-d6cdpdqmxzz5-np-ne-xhfvnf--managed/providers/Microsoft.Compute/virtualMachines/h9k2r7c7n2q4l9a-npnoedge-4-20-pmpd9-ntn64 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000119c40\u003e: \n    failed to create NodePool np-djrg27: failed waiting for nodepool=\"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\" in resourcegroup=\"rg-candidate-4-20-hwlm5v4jsr9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"name\": \"94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:46:47.162303776Z\",\n      \"endTime\": \"2026-09-30T17:05:53.284012473Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000119c40\u003e: \n    failed to create NodePool np-djrg27: failed waiting for nodepool=\"np-djrg27\" for cluster \"cluster-candidate-4-20-djrg27\" in resourcegroup=\"rg-candidate-4-20-hwlm5v4jsr9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"name\": \"94e00bc9-969c-49a7-a690-ac544d6d6b44\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:46:47.162303776Z\",\n      \"endTime\": \"2026-09-30T17:05:53.284012473Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hwlm5v4jsr9p-candidate-djrg27--managed/providers/Microsoft.Compute/virtualMachines/d7d8e3i5j6c6k1p-np-djrg27-4sr5k-bwq7t to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015c0ae0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-twtdf7gdskgb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"name\": \"ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:25:28.987424124Z\",\n      \"endTime\": \"2026-09-30T16:44:30.182558055Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015c0ae0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-twtdf7gdskgb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"name\": \"ffbf36ab-0bd9-4c0c-8698-3035c7937512\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T16:25:28.987424124Z\",\n      \"endTime\": \"2026-09-30T16:44:30.182558055Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-njlb6-5n7mv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-twtdf7gdskgb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-njlb6-5n7mv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","signature_id":"213150af94f11ecc8fc032938950c97f98aae10d4ca8c4c3d158d38b210c0554","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","signature_id":"3933609f5fce2e295de9ab5e9c2e68be690e233f3f4534aede4c689e2581a030","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","signature_id":"4aed755d2532b13cc80a0d2fcd018700a14d4af99ab0fa4f4dfa6f963040d21e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","signature_id":"a26abfcbde513ec1ccfb520656c877d6a0b6d347a97e0f7cca974af4807b4a2b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","signature_id":"ba33ade4a966c1de67f013ea1140c2c38fb60d9c849af3a44a8bb8f1d3d3e42b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","signature_id":"cda87c02afe385de5c6e530de6be078040d84e3fa8634ad1d769024158056676","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","signature_id":"5e2362cdd385e535c236eb25d2af3bedec9c45598cf42305119a1c2199da01c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","signature_id":"d4afdcb13b7abb84b80ed486bbab0e753e39b3cf0759b719778c721a28c9e54b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","signature_id":"d88867851ef4d43076446e92a3c85889999fd11a09b6284afc83108a89d24f58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","signature_id":"f280960e298b057c6eef0496e65821ec4103a9ef2176aedafe051906baf460a7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":4,"after_last_push_count":26,"trend_counts":[3,4,2,1,4,4,4],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00154c140\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-49bq5x\" resourcegroup=\"rg-np-dg-minor-49bq5x-7f6vt9z24c7x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-49bq5x-7f6vt9z24c7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-49bq5x/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00154c140\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-49bq5x\" resourcegroup=\"rg-np-dg-minor-49bq5x-7f6vt9z24c7x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-49bq5x-7f6vt9z24c7x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-49bq5x/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014907c0\u003e: \n    failed to create NodePool np-prg87r: failed starting nodepool creation \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\" in resourcegroup=\"rg-nightly-4-20-kjrkpmc2vcnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kjrkpmc2vcnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-prg87r/nodePools/np-prg87r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0014907c0\u003e: \n    failed to create NodePool np-prg87r: failed starting nodepool creation \"np-prg87r\" for cluster \"cluster-nightly-4-20-prg87r\" in resourcegroup=\"rg-nightly-4-20-kjrkpmc2vcnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kjrkpmc2vcnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-prg87r/nodePools/np-prg87r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b92100\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-qglcmw\" in resourcegroup=\"rg-np-version-upgrade-qglcmw-2xxcc62c56zb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-qglcmw-2xxcc62c56zb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-qglcmw/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b92100\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-qglcmw\" in resourcegroup=\"rg-np-version-upgrade-qglcmw-2xxcc62c56zb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-qglcmw-2xxcc62c56zb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-qglcmw/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","signature_id":"08174c295f58ea65eae7d6210ad569d41e3b1cdc0b0a84a4e29c556f000b91f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","signature_id":"754bee22198c76ca5ecb7193f9b82365056113e3559f87b518bbbd3678c80bd6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","signature_id":"a9ee924e57f9ab9bbddd25402eaa20f29c4967eedecb92e4e71d8d0c06ef84c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","signature_id":"b7ec09633e70cfe3b7246a6bc21475b497dc8e51890bc8b252d77a2143ce919b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,0,2],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00148e9c0\u003e: \n    resource group \"rg-zstream-upgrade-5-0-l6m7gzggxh5z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-mrptx7\" in resourcegroup=\"rg-zstream-upgrade-5-0-l6m7gzggxh5z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"name\": \"d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:28:47.89996524Z\",\n      \"endTime\": \"2026-09-30T13:52:47.080528249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5bj2pp5rm42moqm553gsgpuq5afnl4 still exists (deletion dispatched at 2026-09-30T13:28:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00148e9c0\u003e: \n    resource group \"rg-zstream-upgrade-5-0-l6m7gzggxh5z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-mrptx7\" in resourcegroup=\"rg-zstream-upgrade-5-0-l6m7gzggxh5z\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"name\": \"d96cdefb-0420-40da-ba9f-dde8391c74b5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T13:28:47.89996524Z\",\n      \"endTime\": \"2026-09-30T13:52:47.080528249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t5bj2pp5rm42moqm553gsgpuq5afnl4 still exists (deletion dispatched at 2026-09-30T13:28:50Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a8a4b0\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-b6jrz8\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"name\": \"8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:07:19.758629115Z\",\n      \"endTime\": \"2026-09-30T08:31:25.87901072Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56tcneq332c0ht4fv2g22jot5kdluc still exists (deletion dispatched at 2026-09-30T08:07:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000a8a4b0\u003e: \n    resource group \"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-b6jrz8\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-sxzh6sqdlxpm\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"name\": \"8dac7cbd-9195-4d5c-adc5-24fab76d3fc6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:07:19.758629115Z\",\n      \"endTime\": \"2026-09-30T08:31:25.87901072Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t56tcneq332c0ht4fv2g22jot5kdluc still exists (deletion dispatched at 2026-09-30T08:07:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","signature_id":"06f24328299a032f056108312696270e080fc953f45840834663f37907dc3db6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105187694681264128","occurred_at":"2026-09-30T06:47:13Z","signature_id":"e3db6908a51e323a629ef61d2f3ca60b3da46a872c365d1cca8d13ce3880374f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001116780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-vh7hggp8cw8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\",\n      \"name\": \"eed44f57-1a91-4587-8901-935588d355ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:09:44.769920829Z\",\n      \"endTime\": \"2026-09-30T17:28:44.009160044Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001116780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-vh7hggp8cw8p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eed44f57-1a91-4587-8901-935588d355ba\",\n      \"name\": \"eed44f57-1a91-4587-8901-935588d355ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T17:09:44.769920829Z\",\n      \"endTime\": \"2026-09-30T17:28:44.009160044Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-vh7hggp8cw8p--managed/providers/Microsoft.Compute/virtualMachines/l0v3x1l5h8o3n9c-np-deletable-cmqdm-h6zzh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0013950a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"name\": \"626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:05:30.277326796Z\",\n      \"endTime\": \"2026-09-30T08:24:30.136180053Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0013950a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6stx4mwz4j4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"name\": \"626e713d-65ca-4af7-8b7c-fccfc5b04f35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T08:05:30.277326796Z\",\n      \"endTime\": \"2026-09-30T08:24:30.136180053Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6stx4mwz4j4k--managed/providers/Microsoft.Compute/virtualMachines/u9g6k2x0j4x3g3x-np-deletable-k7zvr-ljwvr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105324689185312768","occurred_at":"2026-09-30T15:51:35Z","signature_id":"2de55e74f48dd0f13326c20bf872d192fa878f413d1746046060d6f653450349","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105187694681264128","occurred_at":"2026-09-30T06:47:13Z","signature_id":"793343d9f2198c07b4a1defe40164b7665fed5f9625606d1878699fdfd9f02db","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"bdaff9adc205561255863f7f3862c4a4a3cdfd05505c7e10e9a53b9e2959cba5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-srbb2hbvsczt/cluster-nightly-5-1-vxb9dm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1af40\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-vxb9dm: failed waiting for cluster=\"cluster-nightly-5-1-vxb9dm\" in resourcegroup=\"rg-nightly-5-1-srbb2hbvsczt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"name\": \"fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:26:50.189510673Z\",\n      \"endTime\": \"2026-09-30T23:55:57.188151115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-105558 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-srbb2hbvsczt/cluster-nightly-5-1-vxb9dm should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c1af40\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-vxb9dm: failed waiting for cluster=\"cluster-nightly-5-1-vxb9dm\" in resourcegroup=\"rg-nightly-5-1-srbb2hbvsczt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"name\": \"fc96c037-ee80-4be7-a7e9-fa4828e28e0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T23:26:50.189510673Z\",\n      \"endTime\": \"2026-09-30T23:55:57.188151115Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.1.0-0.nightly-multi-2026-09-30-105558 is Partial (want Completed), started 23m56s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105432621902204928","occurred_at":"2026-09-30T23:00:28Z","signature_id":"297ae9530982a4810dfc39b4017f6105e35c27ea06de54d22f389e155e41a619","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":10,"trend_counts":[1,1,2,2,1,1,1],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016d54b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016d54b8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105115529080475648","occurred_at":"2026-09-30T02:00:27Z","signature_id":"959da02202a889de6291751a794c7fb5d01b45f09a2d70f1fa8fe082a480a903","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"83447d3d19023b63c00022eba710d55e1a0090317aba9cb3e48328dc0b521261","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ldfdqgv9q9rv\" to finish creating: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd79a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ldfdqgv9q9rv\" to finish creating: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cd79a0\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-ldfdqgv9q9rv\" to finish creating: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105269033774354432","occurred_at":"2026-09-30T12:10:26Z","signature_id":"b859706588c59cf13ba01690598c1e296b21c66674acdd2387e5df98da83bb15","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}