{"meta":{"start_date":"2026-10-03","end_date":"2026-10-03","start_at":"2026-10-03T00:00:00Z","end_at":"2026-10-04T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T07:51:20Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"raw_occurrences":38,"matched_occurrences":37,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":23,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":23,"after_last_push_count":78,"trend_counts":[0,0,0,0,8,47,23],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34ef387fcea0\u003e: \n    failed to create NodePool np-vgspf2: failed waiting for nodepool=\"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\" in resourcegroup=\"rg-nightly-4-21-dsfs76js9854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n      \"name\": \"78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T23:34:31.068400483Z\",\n      \"endTime\": \"2026-10-03T23:53:40.358231747Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: Creating: virtualmachine creating or updating\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x34ef387fcea0\u003e: \n    failed to create NodePool np-vgspf2: failed waiting for nodepool=\"np-vgspf2\" for cluster \"cluster-nightly-4-21-vgspf2\" in resourcegroup=\"rg-nightly-4-21-dsfs76js9854\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n      \"name\": \"78a87535-b0bc-47d2-9812-340ab7a0fb8e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T23:34:31.068400483Z\",\n      \"endTime\": \"2026-10-03T23:53:40.358231747Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-dh299: Creating: virtualmachine creating or updating\\nMachine r3x3m3o0p2b4e4x-np-vgspf2-qj7bv-jlf82: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31a06c9a5d00\u003e: \n    failed to create NodePool np-dz68ws: failed waiting for nodepool=\"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\" in resourcegroup=\"rg-nightly-5-0-4t5m8n4czrvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n      \"name\": \"2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T23:35:55.566084896Z\",\n      \"endTime\": \"2026-10-03T23:55:00.222357543Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: Creating: virtualmachine creating or updating\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x31a06c9a5d00\u003e: \n    failed to create NodePool np-dz68ws: failed waiting for nodepool=\"np-dz68ws\" for cluster \"cluster-nightly-5-0-dz68ws\" in resourcegroup=\"rg-nightly-5-0-4t5m8n4czrvj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n      \"name\": \"2e4aeefa-8e4c-446e-bbb2-afff95c12936\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T23:35:55.566084896Z\",\n      \"endTime\": \"2026-10-03T23:55:00.222357543Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-95fl8: Creating: virtualmachine creating or updating\\nMachine h3s7g6c6v2v9l8v-np-dz68ws-76lrt-kc94l: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x366855210620\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4c8kl482c2st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\",\n      \"name\": \"37548aa4-a4e7-458d-b686-fb1201996386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T23:15:28.253854505Z\",\n      \"endTime\": \"2026-10-03T23:34:30.095088976Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: Creating: virtualmachine creating or updating\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x366855210620\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-4c8kl482c2st\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/37548aa4-a4e7-458d-b686-fb1201996386\",\n      \"name\": \"37548aa4-a4e7-458d-b686-fb1201996386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T23:15:28.253854505Z\",\n      \"endTime\": \"2026-10-03T23:34:30.095088976Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-n2nhw: Creating: virtualmachine creating or updating\\nMachine c4o5k1a3s4v8w4p-np-1-gjcb5-xsvvt: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"390cd3c454160d58c3f66648e63a7907c2041ab6f7cec42614cfb83642a9075a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"5c797d19c80475dc8cd016a2f12b5969e5f66674a64c327fd5b8176cbb909c6a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"88bfecb223eed4d7950743d5a839fc1088854870e36ffeb70cfcf0e0e832d4ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"d2485400b9fb14ab84f4a84285253177801b00cc9d4149c5818d0c29ddef9be2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"e9efc6e08915eb27cce0e9b2aee48e6761e029a06f9f351f49af927ae38ed282","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"f9761523403367d45961176e4cd187bb59f7a5f7a10bfd1b755e1c02f0042a38","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"2c2caa46b0c9a860c8536cb1d0cfe4dc435fe9c87395f2009a2b50d36e765530","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"2c9f39e0ab026a6227defad21130e9e9d394ff6a055a7d1fa925b7fc0262141f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"6ed585f3f050ad8b54d850a6ae0fdd9b4909a19d9c493dbc159c6d4911096dd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"8ed18e43fabdf08315daac2d8e02c94f2c8a172b70d38853c874dbc955f5d982","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"9528ad897be3c780f7f61e03b1ee8a6fe0b00f376a6ff34c98cd8ee75346769c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"e5c35e2b9b861c774ee1789f184fbf1cb2f5f308d8dc05c180346b85e646fe05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"e9fa9ae8e858a7b5355cb1b43d0d6779abb4d6d970a6d15f0440c7b03e009e78","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"fd808e83579b395ae5abd54c9de0aa37f2d5bc4ae6a53ba2e695ba0fa5c2dbc8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"06c296f07902755728beb3838eb1ec4ac170f38778b421b87c5fc204e64d1fe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"173a7e9edaf6f61e13589f056617df6ddb63fa543ce96dbe851cba6372717a02","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"2270ab4d2942cd093aff353fd34037a69c93e335ec7f1a88dfa214a3f4004534","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"2a69c285def7ab9febd458730c74909c9a16be32934cbdbd5ca44844262b1eb0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"34a4dddd4ac406094ca0f1a4bf7eb674580835c68e05a723aea1b94befa47471","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"38fdd980bf52332969e2e4f977878b2b3caffd7b67413af3dedd10df203bfc1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"399671fb721e085227d1bcef5acf1cf7744be28e89d36f981b42cc7c1ec8a4d7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"8f5972671bc46aef70b454f5d37eb1f821ffe511b2716373992fdcfdd6e793dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"ed4c47b0094dced5127989ca5fda7f6a4352c3e90ee689cf14e5def8764d9f1d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":5,"after_last_push_count":37,"trend_counts":[0,5,5,0,0,2,5],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x36252fbe4930\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-548xlmq4vp84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x36252fbe4930\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-548xlmq4vp84/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x336e50b8a360\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-njsbz992h4xx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x336e50b8a360\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-njsbz992h4xx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3102b57c8300\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-5xj4fb5fc6dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3102b57c8300\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-5xj4fb5fc6dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"0c43de85fc2d6a7c071c55d1f96008747d85103a5e49f90377c2b68757383d44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"1c86c022cf7d0c7808be0cd45d82052e35c88c4c35fb78fb4a2b9cf8b10f84d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"4220fe7603791b9ddfb09840fb8cd4a59017f25f10351d3998d45dda9b1f0d93","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"9a1223f045ad2c43a2d2b9f5c9d15450c0250574d5337dd1a988d4371b014517","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"e723e28bcd7e435dcc3421287fec024c49aa82e18f5dc42262402d55af5c17bd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":12,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x104897805d20\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r8db5d4ldfmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n      \"name\": \"12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T23:41:40.611562032Z\",\n      \"endTime\": \"2026-10-04T00:00:46.799966407Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: Creating: virtualmachine creating or updating\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x104897805d20\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r8db5d4ldfmh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n      \"name\": \"12e347fe-1273-4e0c-9b2a-1c191d6d4055\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T23:41:40.611562032Z\",\n      \"endTime\": \"2026-10-04T00:00:46.799966407Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-hlwvx: Creating: virtualmachine creating or updating\\nMachine o0s0h4b0m0k9t7l-np-delete-main-sj9cl-xbp6q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x933c5f12a60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lbkmsp45kpxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\",\n      \"name\": \"efe85861-3de4-4231-94f4-c87043439307\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T03:54:52.260057343Z\",\n      \"endTime\": \"2026-10-03T04:13:53.608229594Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: Creating: virtualmachine creating or updating\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x933c5f12a60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lbkmsp45kpxz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/efe85861-3de4-4231-94f4-c87043439307\",\n      \"name\": \"efe85861-3de4-4231-94f4-c87043439307\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T03:54:52.260057343Z\",\n      \"endTime\": \"2026-10-03T04:13:53.608229594Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-hcvpc: Creating: virtualmachine creating or updating\\nMachine k7l8y6q4q2l9i3h-np-delete-main-68dll-mgh9q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xca0cd836f80\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n      \"name\": \"4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T01:51:49.926221366Z\",\n      \"endTime\": \"2026-10-03T02:10:56.26372719Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: Creating: virtualmachine creating or updating\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xca0cd836f80\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-xrjvzm8fxrgt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n      \"name\": \"4e0749eb-04f8-4fdd-b9be-8393584f3697\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T01:51:49.926221366Z\",\n      \"endTime\": \"2026-10-03T02:10:56.26372719Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-45vn5: Creating: virtualmachine creating or updating\\nMachine m9l6j7a1w6x8b0s-np-deletable-z98x4-bswmx: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"2b24f7ee1c45683523e52ab6a6fd28bf4368385c2a3884bc30ecbb5e5516b086","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"882fba6cad22093956999ee2384f3228d45b1777a330537ec945b37008289a0a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"d6f5de42eefb7152643e950681559e711013d084e28baf1d882e6f029c0e5511","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":50,"anchor_occurrences":2,"after_last_push_count":44,"trend_counts":[1,2,1,10,6,5,2],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":23,"prior_last_seen_at":"2026-09-27T02:00:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15f1e13c9ee0\u003e: \n    failed to create NodePool np-6tqvsp: failed waiting for nodepool=\"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\" in resourcegroup=\"rg-candidate-4-21-qxp54ksqdkvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\",\n      \"name\": \"cd90e427-e90a-4449-9d9c-98919a3e2323\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T03:23:37.870437581Z\",\n      \"endTime\": \"2026-10-03T03:42:43.397028044Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: Creating: virtualmachine creating or updating\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15f1e13c9ee0\u003e: \n    failed to create NodePool np-6tqvsp: failed waiting for nodepool=\"np-6tqvsp\" for cluster \"cluster-candidate-4-21-6tqvsp\" in resourcegroup=\"rg-candidate-4-21-qxp54ksqdkvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/cd90e427-e90a-4449-9d9c-98919a3e2323\",\n      \"name\": \"cd90e427-e90a-4449-9d9c-98919a3e2323\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T03:23:37.870437581Z\",\n      \"endTime\": \"2026-10-03T03:42:43.397028044Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-cm8m4: Creating: virtualmachine creating or updating\\nMachine y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-qxp54ksqdkvb-candidate-6tqvsp--managed/providers/Microsoft.Compute/virtualMachines/y1i0m6w3c6r9w2i-np-6tqvsp-mg8xh-88lr5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbbf668ab320\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lqwbkg\" in resourcegroup=\"rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n      \"name\": \"a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T01:58:23.770970582Z\",\n      \"endTime\": \"2026-10-03T02:17:26.319658092Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbbf668ab320\u003e: \n    failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-lqwbkg\" in resourcegroup=\"rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n      \"name\": \"a4143c12-1ac1-4d68-ba18-1bb777cf81b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T01:58:23.770970582Z\",\n      \"endTime\": \"2026-10-03T02:17:26.319658092Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lqwbkg-jl7ktvcrp5z8-np-upgrade-lq-e094d963/providers/Microsoft.Compute/virtualMachines/q0g7j2k6j1j2c5g-npupgrade-4-21-z7zbc-cgvzh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"8dc98ddc487d792e30d36e88cd34b10e8b904d76ef343c1f573a64f712f3b220","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"d5c37afd72eeef3b1717f0ed626a3a3f20e451ad1fbcbdcf4ca6127c60afb433","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":2,"after_last_push_count":30,"trend_counts":[1,4,4,4,0,2,2],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x242124721260\u003e: \n    failed to create NodePool np-zb9djq: failed starting nodepool creation \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\" in resourcegroup=\"rg-nightly-4-20-r55xk7zmjr9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-r55xk7zmjr9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zb9djq/nodePools/np-zb9djq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x242124721260\u003e: \n    failed to create NodePool np-zb9djq: failed starting nodepool creation \"np-zb9djq\" for cluster \"cluster-nightly-4-20-zb9djq\" in resourcegroup=\"rg-nightly-4-20-r55xk7zmjr9c\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-r55xk7zmjr9c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zb9djq/nodePools/np-zb9djq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0x184038744400\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tzvzfh\" in resourcegroup=\"rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tzvzfh/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0x184038744400\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-tzvzfh\" in resourcegroup=\"rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-tzvzfh-8sk9jqkdk6gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-tzvzfh/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"4e5c1277a0285514fbe13ffd0e118b84890287d84121f73af4e99275508596ce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106519855258669056","occurred_at":"2026-10-03T23:00:45Z","signature_id":"80a54d00d33387405efc62cd65294145e6838630c336a95f7908f0dc39f21afd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":7,"trend_counts":[0,0,0,0,1,5,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1363e14cc018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-jtmvgv9m6829\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\",\n      \"name\": \"31894849-27e9-4053-a67e-9e3babf2ad8a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T01:01:05.764058109Z\",\n      \"endTime\": \"2026-10-03T01:20:12.852653102Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1363e14cc018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-jtmvgv9m6829\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31894849-27e9-4053-a67e-9e3babf2ad8a\",\n      \"name\": \"31894849-27e9-4053-a67e-9e3babf2ad8a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T01:01:05.764058109Z\",\n      \"endTime\": \"2026-10-03T01:20:12.852653102Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wxgnc-9w4jt: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wxgnc-clcm7: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106178703519125504","occurred_at":"2026-10-03T00:25:08Z","signature_id":"475946b6879259ac2394beb241c800f706ea9c7b18c2b17cfb61377bbdc2cf11","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-27..2026-10-03 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xe7412ec40a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-j2962r94fd6x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n      \"name\": \"4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T03:00:19.581988952Z\",\n      \"endTime\": \"2026-10-03T03:19:19.770602537Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xe7412ec40a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-j2962r94fd6x\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n      \"name\": \"4588d6e6-e963-44fd-8b18-89f83b2a899d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-03T03:00:19.581988952Z\",\n      \"endTime\": \"2026-10-03T03:19:19.770602537Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2106202761350615040","occurred_at":"2026-10-03T02:00:44Z","signature_id":"39751860e6d26d74a5f620ec93a78ea99d0e96ecd0ed7587dea64af7ba46a985","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}