{"meta":{"start_date":"2026-10-07","end_date":"2026-10-07","start_at":"2026-10-07T00:00:00Z","end_at":"2026-10-08T00:00:00Z","timezone":"UTC","generated_at":"2026-10-30T11:20:41Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":6,"raw_occurrences":61,"matched_occurrences":60,"runs_affected":5},"rows":[{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":40,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":40,"after_last_push_count":104,"trend_counts":[6,5,2,1,1,18,40],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-21","2026-09-28"],"prior_runs_affected":32,"prior_last_seen_at":"2026-10-04T02:00:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xkh7k5\" for cluster \"cluster-nightly-4-22-xkh7k5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e843b97f120\u003e: \n    failed to create NodePool np-xkh7k5: failed waiting for nodepool=\"np-xkh7k5\" for cluster \"cluster-nightly-4-22-xkh7k5\" in resourcegroup=\"rg-nightly-4-22-lnvbhdz2bflf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\",\n      \"name\": \"dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:40:08.180826205Z\",\n      \"endTime\": \"2026-10-08T00:09:12.758937229Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc to exist\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc to exist\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xkh7k5\" for cluster \"cluster-nightly-4-22-xkh7k5\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e843b97f120\u003e: \n    failed to create NodePool np-xkh7k5: failed waiting for nodepool=\"np-xkh7k5\" for cluster \"cluster-nightly-4-22-xkh7k5\" in resourcegroup=\"rg-nightly-4-22-lnvbhdz2bflf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\",\n      \"name\": \"dc30ffaf-307f-4841-b4a2-6d2c8334bf3b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:40:08.180826205Z\",\n      \"endTime\": \"2026-10-08T00:09:12.758937229Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc to exist\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-8jjmc to exist\\nMachine a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-lnvbhdz2bflf-nightly-xkh7k5--managed/providers/Microsoft.Compute/virtualMachines/a2v2x5b8f7z8j6l-np-xkh7k5-rx767-sm68r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a93c5372000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-v2qsz6wkc724\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93d34adc-e5de-4e29-8488-7bce9a8a626f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93d34adc-e5de-4e29-8488-7bce9a8a626f\",\n      \"name\": \"93d34adc-e5de-4e29-8488-7bce9a8a626f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:29.82535222Z\",\n      \"endTime\": \"2026-10-07T23:46:38.16324507Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-mp2gq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-mp2gq to exist\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-rhh25: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-rhh25 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-mp2gq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-mp2gq to exist\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-rhh25: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-rhh25 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a93c5372000\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-v2qsz6wkc724\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93d34adc-e5de-4e29-8488-7bce9a8a626f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93d34adc-e5de-4e29-8488-7bce9a8a626f\",\n      \"name\": \"93d34adc-e5de-4e29-8488-7bce9a8a626f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:29.82535222Z\",\n      \"endTime\": \"2026-10-07T23:46:38.16324507Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-mp2gq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-mp2gq to exist\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-rhh25: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-rhh25 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-mp2gq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-mp2gq to exist\\nMachine u2g6d8v8s1g6k8i-np-1-szw76-rhh25: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-v2qsz6wkc724--managed/providers/Microsoft.Compute/virtualMachines/u2g6d8v8s1g6k8i-np-1-szw76-rhh25 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dhdr4d\" for cluster \"cluster-nightly-5-0-dhdr4d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bb2d5ca8760\u003e: \n    failed to create NodePool np-dhdr4d: failed waiting for nodepool=\"np-dhdr4d\" for cluster \"cluster-nightly-5-0-dhdr4d\" in resourcegroup=\"rg-nightly-5-0-w2nf95ndpzrw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5845b729-520b-4e46-b237-ae83e6fffeab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5845b729-520b-4e46-b237-ae83e6fffeab\",\n      \"name\": \"5845b729-520b-4e46-b237-ae83e6fffeab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:07.401401662Z\",\n      \"endTime\": \"2026-10-08T00:12:12.648695446Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk to exist\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk to exist\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dhdr4d\" for cluster \"cluster-nightly-5-0-dhdr4d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1bb2d5ca8760\u003e: \n    failed to create NodePool np-dhdr4d: failed waiting for nodepool=\"np-dhdr4d\" for cluster \"cluster-nightly-5-0-dhdr4d\" in resourcegroup=\"rg-nightly-5-0-w2nf95ndpzrw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5845b729-520b-4e46-b237-ae83e6fffeab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5845b729-520b-4e46-b237-ae83e6fffeab\",\n      \"name\": \"5845b729-520b-4e46-b237-ae83e6fffeab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:07.401401662Z\",\n      \"endTime\": \"2026-10-08T00:12:12.648695446Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk to exist\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-n9bkk to exist\\nMachine p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-w2nf95ndpzrw-nightly-dhdr4d--managed/providers/Microsoft.Compute/virtualMachines/p2s4e8y8s1w9m5p-np-dhdr4d-mg9mh-qzh8j to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"0d3977ea4a90af04720c42d0bb759a3fb96691ce43398b60e6eef15c0a367212","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"35c1bfad26ec391bd6694018ed5a8d59c0813fc87ac12982198a869b9604acb1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"68cb7350eeb61710d23cb67024cbbf9383969e0663d83204c8bdbdff0d33b967","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"698107089cf97b784786a6213e09a0a68dc2d0bd01e69c8e0e0c0fdb8f948aab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"8ee2deee6d590c8c6f98a003b663ac08a67a9a1ecc4cc6560bd842c5b1c9a52d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"a0dba5d1188bd276353fe4d2637a368162095ec26a8c24aee58b4ef68e6b2355","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"d3f83bccde47a3e6aa4acd1828dd9099b38db2477a89ab6bee0285d81eb16b5f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"01163b61c17b562a500c04958cb33fe485d2c7ca115bc405d91028b41fee31fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"30c2dae7bbb702188aef5ab4608ef49e36dd251396ec6996c25febee6fad61be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"62b9a0dbb94dec0c48d4230d8c8526f7eb93e53eaeef32b48228dd8d42af77f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"6e47728030dcf1f6d034b909c5fda78d2f12b066fb41e2f513f61773969a2a38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"70ad7e4081cf1f599d09c2f83be92ab81460d5e3baf40247b0aaf860738f9066","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"bd4230f06a85eb9e1292a6291f9776d671fb62ec21c099d1d8dbdbd83275c152","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"cf1becd17e665f40e312fb2fb9948d50c9c5fa62d4421b6733acabe9bde7f248","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"ede7e9813082bc8cecccb538add5f1a3c53fe9bae98639f390f3a1e3e9e834e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"0295e7035dfce1af760e5a9f96d8886860036c81cb7a349f83dfe84f97f0ab25","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"2efd6126d606d5f6f80d0316433b168030b43d3ea235668b13059c9b830d3aa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"660518fdfbb9f0800a819d55d8bc3171022d1eadfaf9a16981cf1f090bdaf04c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"844a548f21a121190f4b76ba3514e970ff80d7128375752094c4e416f2fd6f61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"88363a2b12377ced6544609973aca118fa1d1df45bf3c3b0605e39c27653faec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"8a41ca55dc9a29f3bb19bad80e28580eee3b1db49cbcab92f2d225a074105caf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"a131321bd10141e15cfba687cd99d26f8cc540813c1b560df836c2ddf028dca9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"c1fc956072a43b79a620c5e7511a401ac3079dfd2ffe4780e4315cf518f89d75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"ffa11b5d4792edceb448fa6645e938d4beba1a8400ac52ce970668a6ebf44a09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"42d95229988ca74bd7cb337361282d8c16d787466292deaf2232070c3cef245e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"636c0e8426e9e41af610392dfe2585499dc4ca5ac63cc2c2ad03af827766f74a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"6afa453b179354544c5915baf6f73ad97a3103ed41f793200617265657041048","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"ace0a1418b7953740f38672a49af405998a9e3e24c4a33b95e0d999dd68228b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"b2b0c546047cd7f8dfbc1e8fd084cbf9b0c93bccae10e310d13b26a94be9f023","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"d41cf6f362daae9f0f271e170de364045dff8bca3bf030691011bad678666f7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"def278b028e2cfc71409628aa852a649d7df162da0e1ca879d80fffe9d9e1dd1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"f6435cbef4dc6ebe4912bc5681a94ef53c08760658835c2e2be55f09f269435c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"0af96c9de788a43bfc36cc28d4ef2f7ed16e382ed3dce340c647a636af5ecb10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"2d1db9807c28b66174164739cafde112b3198801994efbfaf7eb91ee2bc663c3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"461d3cd85376afaa0deb74d80a0b8c0bf44e6e13b152912c7fa7055c0a01ba52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"4acdd568139e3c3c134edd0d6f256da22e4a7a8368cc84137523b794cc84f751","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"87cadb376d6cd5fe3148fe99a8a0a312ae518ce1cd8d34338748d862f979c65e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"90e197c419ca4c3d130ecefd2a74215194ffd5df0642d2dc2d4c34686fc824f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"9264fae13d8a0e126024fe87016ca0eb955eeacc351b1d83a0e62b7b2d4f8888","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"e3312dc69a51a671f116346996b6b0e62dfdfde1bc0d102af6a4d41fef2b5576","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9d76b8574a881c48ca814c73a5f78cfd9c34fbf0c2d12d09c9f9b6e856adedcf","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":8,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":8,"after_last_push_count":9,"trend_counts":[0,0,0,0,1,0,8],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26a27898a0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m9xrgvjx8lwn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"name\": \"e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:33.092346932Z\",\n      \"endTime\": \"2026-10-07T23:46:38.337633525Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x26a27898a0d8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-m9xrgvjx8lwn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"name\": \"e7fd4c79-ee4a-4813-89bb-a9a38553de85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:17:33.092346932Z\",\n      \"endTime\": \"2026-10-07T23:46:38.337633525Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-whw2v-lppqg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-lppqg to exist\\nMachine cilium-cluster-cilium-np-whw2v-x7t8r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-m9xrgvjx8lwn--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-whw2v-x7t8r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x390344d20c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4wvdmxwppl4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\",\n      \"name\": \"02042050-d676-4d90-bee1-de9c86994c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:29.815037524Z\",\n      \"endTime\": \"2026-10-07T14:59:36.125232922Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x390344d20c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4wvdmxwppl4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/02042050-d676-4d90-bee1-de9c86994c80\",\n      \"name\": \"02042050-d676-4d90-bee1-de9c86994c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:29.815037524Z\",\n      \"endTime\": \"2026-10-07T14:59:36.125232922Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-56qxc-gqmsv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-gqmsv to exist\\nMachine cilium-cluster-cilium-np-56qxc-sft5q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-4wvdmxwppl4j--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-56qxc-sft5q to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xb98d7c3a108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6z9fvf852z99\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93cf84b2-52d0-403e-b5be-b66df90e2622\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93cf84b2-52d0-403e-b5be-b66df90e2622\",\n      \"name\": \"93cf84b2-52d0-403e-b5be-b66df90e2622\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:19.676644212Z\",\n      \"endTime\": \"2026-10-07T14:59:26.138957659Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-m8jkv-j7ddl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6z9fvf852z99--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m8jkv-j7ddl to exist\\nMachine agg-cluster-agg-np-a-m8jkv-lsbmq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-m8jkv-lsbmq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-m8jkv-j7ddl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6z9fvf852z99--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m8jkv-j7ddl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xb98d7c3a108\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-6z9fvf852z99\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93cf84b2-52d0-403e-b5be-b66df90e2622\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93cf84b2-52d0-403e-b5be-b66df90e2622\",\n      \"name\": \"93cf84b2-52d0-403e-b5be-b66df90e2622\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T14:30:19.676644212Z\",\n      \"endTime\": \"2026-10-07T14:59:26.138957659Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-m8jkv-j7ddl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6z9fvf852z99--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m8jkv-j7ddl to exist\\nMachine agg-cluster-agg-np-a-m8jkv-lsbmq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-m8jkv-lsbmq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-m8jkv-j7ddl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6z9fvf852z99--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m8jkv-j7ddl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"6ba651d962b63ca9737c83935c51fabd9671a4349db5bc958c0be99ef420497d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"38e91e14bbfa3b78d5e999c68fea443396976fc0d1daad98d4cf67b9eaa12e1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"f9b070f99215534333fb93adb09f749de178f54c1544c512656282297ceb7a23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"f9af646ef53adca9c38d8c7a82da5f67f58665aa87755f378eaeda7b1930de7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"255e1dca1572a70b8f0d0f5f6aa345989e66a6b4f6f9e32911b805b3bfa7a379","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"c77d4d78f4573df1f5f58548e67fdb600dd43ca5bb56e96677224027c975bc44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"06edfc93dd116cf1f2412df4933dab83c6d60f345bd6545be6187369b744fbb9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"9174db2c962c5b14b002b9d9fc7ad502cc024cdfc29401bc9447ace098a0ae20","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bxmr827d8pqp\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":5,"after_last_push_count":5,"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-28"],"prior_runs_affected":24,"prior_last_seen_at":"2026-10-04T22:01:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x115d2dac76c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-knjdfr9bnmx4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"name\": \"3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:45:51.426276166Z\",\n      \"endTime\": \"2026-10-08T00:14:52.690397112Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x115d2dac76c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-knjdfr9bnmx4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"name\": \"3365f6a5-3dc7-4c04-bf39-d10dc605849e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:45:51.426276166Z\",\n      \"endTime\": \"2026-10-08T00:14:52.690397112Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-vn2nz to exist\\nMachine u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-knjdfr9bnmx4--managed/providers/Microsoft.Compute/virtualMachines/u0h1u4p4y0s3z3s-np-deletable-2zzc7-xpmzt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x25e8dd52bee0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tkm2qrpq9mx2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b4716e73-b072-461b-9a76-d43c0f1198b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b4716e73-b072-461b-9a76-d43c0f1198b7\",\n      \"name\": \"b4716e73-b072-461b-9a76-d43c0f1198b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T15:25:44.42195423Z\",\n      \"endTime\": \"2026-10-07T15:54:45.869722515Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97 to exist\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97 to exist\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x25e8dd52bee0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-tkm2qrpq9mx2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b4716e73-b072-461b-9a76-d43c0f1198b7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b4716e73-b072-461b-9a76-d43c0f1198b7\",\n      \"name\": \"b4716e73-b072-461b-9a76-d43c0f1198b7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T15:25:44.42195423Z\",\n      \"endTime\": \"2026-10-07T15:54:45.869722515Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97 to exist\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-5nn97 to exist\\nMachine q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-tkm2qrpq9mx2--managed/providers/Microsoft.Compute/virtualMachines/q5l6c5w9a6d0j2h-np-delete-main-vkt28-fd6j4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x28b27f4fd8c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-flmg4dn4tndz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8023a4bb-83f8-470d-8d09-d112e1caa401\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8023a4bb-83f8-470d-8d09-d112e1caa401\",\n      \"name\": \"8023a4bb-83f8-470d-8d09-d112e1caa401\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:07:13.045361342Z\",\n      \"endTime\": \"2026-10-07T11:36:15.865514346Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz to exist\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz to exist\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x28b27f4fd8c0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-flmg4dn4tndz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8023a4bb-83f8-470d-8d09-d112e1caa401\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8023a4bb-83f8-470d-8d09-d112e1caa401\",\n      \"name\": \"8023a4bb-83f8-470d-8d09-d112e1caa401\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:07:13.045361342Z\",\n      \"endTime\": \"2026-10-07T11:36:15.865514346Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz to exist\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-9sqcz to exist\\nMachine v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-flmg4dn4tndz--managed/providers/Microsoft.Compute/virtualMachines/v2z1k6a9o7e9k0r-np-deletable-qwwz5-bknr5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"41911d09cb3faeb31e1614a3ce95b71553ee40f58fbdbb9b2b3538c30fc780cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"e1898cdee251c0de6f399c876a928fe1ac01341d45cd38e4d6b00e9b3fb43608","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107763728446394368","occurred_at":"2026-10-07T09:23:26Z","signature_id":"fbe36cbbceb1d0e1025eb3b54fd10fe074a1d442f305fcf1dd1cb97792c28108","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"c701b9c1b5bb023dff9bd4abb1de8fbc2c5ce5288ba7fb66eaf94d47e43c2e87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2107652362595209216","occurred_at":"2026-10-07T02:00:56Z","signature_id":"6cb55ad6e469d73beffb431fa7679fc0430ca42fd9f6b8f1a796395bc2d38f01","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":5,"after_last_push_count":52,"trend_counts":[0,2,5,0,5,5,5],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":3,"prior_window_starts":["2026-09-14","2026-09-21","2026-09-28"],"prior_runs_affected":11,"prior_last_seen_at":"2026-10-03T23:00:45Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x27f249e4c7e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-w7l7gcf22jkn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x27f249e4c7e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-w7l7gcf22jkn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1512977bb1a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qrbwmhc6tvhs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1512977bb1a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qrbwmhc6tvhs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3dbe3c0729c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-g2tcvfj4mtzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3dbe3c0729c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-g2tcvfj4mtzb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"0660a8b07b67f6039cafbd2db5de6076302394cf9033181aa61d0e1bed619a61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"24834a656a673fd3cfb5ae1fccf25d73d8b576895e93c65a3c6a94a528913c38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"748728f92e96c01a73c8dc01210faf4d617f86fb186b83b70b3031c704b1a940","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"79bf72a228ccd77d5a3ed3ea923d269a222d78760e0d3456d70db4a2a4ed54ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2107969374252961792","occurred_at":"2026-10-07T23:00:37Z","signature_id":"abb5a95e8ea83117472e6c379d204e8f2c25f4bdb19bf5b0daa764dda033af3b","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"5d4aaaaae5c748327f53adcfa681b6d36f3f8da08e1b106abec14dbcf1f183c1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down cluster resources, managed cluster, hosted cluster, namespaces; [descendantResources] remaining resources: applydesires, externalAuths, nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x27901568a690\u003e: \n    resource group \"external-auth-cluster-ch86qj77dnn4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ch86qj77dnn4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"name\": \"0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T06:22:44.810261014Z\",\n      \"endTime\": \"2026-10-07T06:46:52.177879454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x279015820520\u003e{\n                msg: \"resource group \\\"external-auth-cluster-ch86qj77dnn4\\\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0x279015820460\u003e{\n                    msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*fmt.wrapError | 0x279015820440\u003e{\n                        msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                        err: \u003c*fmt.wrapError | 0x279015820420\u003e{\n                            msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            err: \u003c*exported.ResponseError | 0x27901606a960\u003e{\n                                ErrorCode: \"InternalServerError\",\n                                StatusCode: 200,\n                                RawResponse: {\n                                    Status: \"200 OK\",\n                                    StatusCode: 200,\n                                    Proto: \"HTTP/2.0\",\n                                    ProtoMajor: 2,\n                                    ProtoMinor: 0,\n                                    Header: {\n                                        \"Cache-Control\": [...],\n                                        \"X-Envoy-Upstream-Service-Time\": [...],\n                                        \"X-Ms-Operation-Identifier\": [...],\n                                        \"X-Ms-Correlation-Request-Id\": [...],\n                                        \"X-Msedge-Ref\": [...],\n                                        \"Pragma\": [...],\n                                        \"Content-Type\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [...],\n                                        \"Strict-Transport-Security\": [...],\n                                        \"X-Content-Type-Options\": [...],\n                                        \"Date\": [...],\n                                        \"Content-Length\": [...],\n                                        \"X-Ms-Request-Id\": [...],\n                                        \"X-Ms-Routing-Request-Id\": [...],\n                                        \"X-Cache\": [...],\n                                        \"Expires\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [...],\n                                    },\n                                    Body: \u003c*exported.nopClosingBytesReader | 0x279015820220\u003e{\n                                        s: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        i: 0,\n                                    },\n                                    ContentLength: 1273,\n                                    TransferEncoding: nil,\n                                    Close: false,\n                                    Uncompressed: false,\n                                    Trailer: nil,\n                                    Request: {\n                                        Method: \"GET\",\n                                        URL: {Scheme: ..., Opaque: ..., User: ..., Host: ..., Path: ..., Fragment: ..., RawQuery: ..., RawPath: ..., RawFragment: ..., ForceQuery: ..., OmitHost: ...},\n                                        Proto: \"HTTP/1.1\",\n                                        ProtoMajor: 1,\n                                        ProtoMinor: 1,\n                                        Header: {...: ..., ...: ...},\n                                        Body: nil,\n                                        GetBody: nil,\n                                        ContentLength: 0,\n                                        TransferEncoding: nil,\n                                        Close: false,\n                                        Host: \"management.azure.com\",\n                                        Form: nil,\n                                        PostForm: nil,\n                                        MultipartForm: nil,\n                                        Trailer: nil,\n                                        RemoteAddr: \"\",\n                                        RequestURI: \"\",\n                                        TLS: nil,\n                                        Cancel: nil,\n                                        Response: nil,\n                                        Pattern: \"\",\n                                        ctx: \u003csuppressed context\u003e,\n                                        pat: nil,\n                                        matches: nil,\n                                        otherValues: nil,\n                                    },\n                                    TLS: {\n                                        Version: 772,\n                                        HandshakeComplete: true,\n                                        DidResume: false,\n                                        CipherSuite: 4866,\n                                        CurveID: 25,\n                                        NegotiatedProtocol: \"h2\",\n                                        NegotiatedProtocolIsMutual: true,\n                                        ServerName: \"management.azure.com\",\n                                        PeerCertificates: [..., ..., ...],\n                                        VerifiedChains: [...],\n                                        SignedCertificateTimestamps: nil,\n                                        OCSPResponse: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        TLSUnique: nil,\n                                        ECHAccepted: false,\n                                        HelloRetryRequest: true,\n                                        ekm: 0x585e900,\n                                        testingOnlyPeerSignatureAlgorithm: 2052,\n                                    },\n                                },\n                                errMsg: \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0x27901568a690\u003e: \n    resource group \"external-auth-cluster-ch86qj77dnn4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-ch86qj77dnn4\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"name\": \"0cf8be7c-97b3-497d-b733-73f64068db87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T06:22:44.810261014Z\",\n      \"endTime\": \"2026-10-07T06:46:52.177879454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x279015820520\u003e{\n                msg: \"resource group \\\"external-auth-cluster-ch86qj77dnn4\\\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                err: \u003c*fmt.wrapError | 0x279015820460\u003e{\n                    msg: \"failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                    err: \u003c*fmt.wrapError | 0x279015820440\u003e{\n                        msg: \"at least one hcp cluster failed to delete: failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                        err: \u003c*fmt.wrapError | 0x279015820420\u003e{\n                            msg: \"failed waiting for hcpCluster=\\\"ea-cluster\\\" in resourcegroup=\\\"external-auth-cluster-ch86qj77dnn4\\\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            err: \u003c*exported.ResponseError | 0x27901606a960\u003e{\n                                ErrorCode: \"InternalServerError\",\n                                StatusCode: 200,\n                                RawResponse: {\n                                    Status: \"200 OK\",\n                                    StatusCode: 200,\n                                    Proto: \"HTTP/2.0\",\n                                    ProtoMajor: 2,\n                                    ProtoMinor: 0,\n                                    Header: {\n                                        \"Cache-Control\": [...],\n                                        \"X-Envoy-Upstream-Service-Time\": [...],\n                                        \"X-Ms-Operation-Identifier\": [...],\n                                        \"X-Ms-Correlation-Request-Id\": [...],\n                                        \"X-Msedge-Ref\": [...],\n                                        \"Pragma\": [...],\n                                        \"Content-Type\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [...],\n                                        \"Strict-Transport-Security\": [...],\n                                        \"X-Content-Type-Options\": [...],\n                                        \"Date\": [...],\n                                        \"Content-Length\": [...],\n                                        \"X-Ms-Request-Id\": [...],\n                                        \"X-Ms-Routing-Request-Id\": [...],\n                                        \"X-Cache\": [...],\n                                        \"Expires\": [...],\n                                        \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [...],\n                                    },\n                                    Body: \u003c*exported.nopClosingBytesReader | 0x279015820220\u003e{\n                                        s: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        i: 0,\n                                    },\n                                    ContentLength: 1273,\n                                    TransferEncoding: nil,\n                                    Close: false,\n                                    Uncompressed: false,\n                                    Trailer: nil,\n                                    Request: {\n                                        Method: \"GET\",\n                                        URL: {Scheme: ..., Opaque: ..., User: ..., Host: ..., Path: ..., Fragment: ..., RawQuery: ..., RawPath: ..., RawFragment: ..., ForceQuery: ..., OmitHost: ...},\n                                        Proto: \"HTTP/1.1\",\n                                        ProtoMajor: 1,\n                                        ProtoMinor: 1,\n                                        Header: {...: ..., ...: ...},\n                                        Body: nil,\n                                        GetBody: nil,\n                                        ContentLength: 0,\n                                        TransferEncoding: nil,\n                                        Close: false,\n                                        Host: \"management.azure.com\",\n                                        Form: nil,\n                                        PostForm: nil,\n                                        MultipartForm: nil,\n                                        Trailer: nil,\n                                        RemoteAddr: \"\",\n                                        RequestURI: \"\",\n                                        TLS: nil,\n                                        Cancel: nil,\n                                        Response: nil,\n                                        Pattern: \"\",\n                                        ctx: \u003csuppressed context\u003e,\n                                        pat: nil,\n                                        matches: nil,\n                                        otherValues: nil,\n                                    },\n                                    TLS: {\n                                        Version: 772,\n                                        HandshakeComplete: true,\n                                        DidResume: false,\n                                        CipherSuite: 4866,\n                                        CurveID: 25,\n                                        NegotiatedProtocol: \"h2\",\n                                        NegotiatedProtocolIsMutual: true,\n                                        ServerName: \"management.azure.com\",\n                                        PeerCertificates: [..., ..., ...],\n                                        VerifiedChains: [...],\n                                        SignedCertificateTimestamps: nil,\n                                        OCSPResponse: [..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ..., ...],\n                                        TLSUnique: nil,\n                                        ECHAccepted: false,\n                                        HelloRetryRequest: true,\n                                        ekm: 0x585e900,\n                                        testingOnlyPeerSignatureAlgorithm: 2052,\n                                    },\n                                },\n                                errMsg: \"GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"name\\\": \\\"0cf8be7c-97b3-497d-b733-73f64068db87\\\",\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"startTime\\\": \\\"2026-10-07T06:22:44.810261014Z\\\",\\n  \\\"endTime\\\": \\\"2026-10-07T06:46:52.177879454Z\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t9p58rkrigg6n9uc8454ajabpln3t9n still exists (deletion dispatched at 2026-10-07T06:22:45Z); [clusterServiceStatus] ClusterService state is \\\\\\\"uninstalling\\\\\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/externalAuths, 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 6 microsoft.redhatopenshift/hcpopenshiftclusters/applydesires; [resourceTeardown] tearing down cluster resources (3), managed cluster (1), hosted cluster (1), namespaces (2)\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107701758112305152","occurred_at":"2026-10-07T05:17:12Z","signature_id":"bbd40a695906a4219c65ee7a3a59b31da7d9339ef823acf986d08e0c6c81eb03","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"88cee2a035a0f46609d44b8db5480aa9ed0efd1c5fec7797c0cbce1364777b87","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-10-01..2026-10-07 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2655580ad6c0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.006s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x2655580ad6c0\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\n    {\n        s: \"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \\\"https://acme-staging-v02.api.letsencrypt.org/directory\\\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:55923-\u003e172.30.0.10:53: read: connection refused\",\n    }"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2107827709018116096","occurred_at":"2026-10-07T13:37:41Z","signature_id":"28108545b283b909283d0a8f87e72879ae1a1db48c35cd0650ff7b4851a09bcb","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}