{"meta":{"start_date":"2026-10-02","end_date":"2026-10-02","start_at":"2026-10-02T00:00:00Z","end_at":"2026-10-03T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T08:56:41Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":6,"failed_runs":6,"raw_occurrences":84,"matched_occurrences":84,"runs_affected":6},"rows":[{"environment":"stg","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":47,"runs_affected":6,"failed_runs":6,"run_impact_percent":100,"anchor_occurrences":47,"after_last_push_count":55,"trend_counts":[0,0,0,0,0,8,47],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":4},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00100c018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dbjg2z24dvcx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n      \"name\": \"2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:17:46.069336916Z\",\n      \"endTime\": \"2026-10-02T23:36:53.678900847Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-p829p-t26hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-p829p-t26hm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00100c018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dbjg2z24dvcx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n      \"name\": \"2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:17:46.069336916Z\",\n      \"endTime\": \"2026-10-02T23:36:53.678900847Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-p829p-t26hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-p829p-t26hm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010647a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-b4pbcj49h2gk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\",\n      \"name\": \"56464cc9-6bea-4036-8793-830b73f9c445\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:18:22.317106803Z\",\n      \"endTime\": \"2026-10-02T23:37:23.768654646Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: Creating: virtualmachine creating or updating\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010647a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-b4pbcj49h2gk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\",\n      \"name\": \"56464cc9-6bea-4036-8793-830b73f9c445\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:18:22.317106803Z\",\n      \"endTime\": \"2026-10-02T23:37:23.768654646Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: Creating: virtualmachine creating or updating\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c80140\u003e: \n    failed to create NodePool np-fpzhbk: failed waiting for nodepool=\"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\" in resourcegroup=\"rg-nightly-4-21-lf2wqrcgx8mk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n      \"name\": \"f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:36:50.854597667Z\",\n      \"endTime\": \"2026-10-02T23:55:53.768834785Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: Creating: virtualmachine creating or updating\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c80140\u003e: \n    failed to create NodePool np-fpzhbk: failed waiting for nodepool=\"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\" in resourcegroup=\"rg-nightly-4-21-lf2wqrcgx8mk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n      \"name\": \"f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:36:50.854597667Z\",\n      \"endTime\": \"2026-10-02T23:55:53.768834785Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: Creating: virtualmachine creating or updating\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"68ded3176e1d7fbcd1f8d33ace4e652b4594754aa12dcb2811b6d01aec17dc6b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"860eb571af757ccc8cd08de2262a7ce14edf5abeb53618b97b6ce441d905813c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"8a7d51d9f85003ad4001a7f59ed5782eb2de4e62df81b0e16178abf5c822b1aa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"8eb7f7f722acb8f7d85fccf2574849a9b3348847ef0469070b23978413484f9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"cd2c3947df5446ae84ba6b5d7e021afccf38c12db58a8bee834290fd0af21ba2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"dfbe2f9b56ec8f589a40188c05855969f4988812afdbce974f8743866b87d6f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"250d803c2dde1e52d60a32aaa2d7a8f52b97c3de220de17e241c3f6720b1dab2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"557730ef8a8a538ac02f9730574bc105fe419054c1342595ab60e5a9b836a55b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"6a6e10696b7a19366e889f690b2ab374029b36eaa72533a7417b96b82b33497e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"abdd7c859f25fcc9e0e3eca9fc927795be2131c3357811538ef9ccf29668a7b0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"d53697eeb27bf8ab55313c9278e2e56de2e878d8f994cba4d1ca76149b70e248","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"df71d9af24f4d024cb085b025d0e7702e69475d8342378c7ed71b98fdfda61a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"f1c205eb5adab7b42be8bb2e5810e6b3d460997323e06bef104b859fa7c1c1fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"0f05a349b9fb3a19a653fac755019ca3729cea73f673115f377ea1ff53ca460a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"1991e4120c3a9404f4a08486e54a3b094270ef79c075deb389ccb4f87dd19186","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"3beeb059ed2eee1c225e5fa6e52e389772213e9dbefa4285fead2b9c61d38ee4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"63e9790a996ec967d16e002977f953d1b88bb5b5420740a8c85e4a4ad4676d27","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"7f3920bc72425d6da0c2994e526b54c2ba9fd9d44a6c8cb1fdea88ed4dae4ca0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"94dbc4ea2092ec4f2334ecba4369c328eb3548419ec0eafdb59df728c566ad14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"c795eac422f14a1d4cf2c9186a4a74089ac3865e0cf6aad349163a72b03db067","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"f9e948ed90e8383bba7bd2c174ac649975b4723ac3ec4ecc8f0a4e9d83325283","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"ff0f4f85405eb058ab255730c9bfcc4dfebb9a6c55cbfe3ed6cc4e8bfb37dbfd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"04d1cb3cb4234161cc07dedd18e19e98c569ee86bfc6884e682791a6475e5f03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"417e38533ea85fbd583e5552fc77a77554d65e5a8dd9d15343bf93a32380c555","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"615a1a8b96dee5eb1bf6319d009a9ff736c86e855049cffe408293ba6acfef53","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"6c3e750da0e9592b1f2dec07e4441b5b99e79bcaca5632e733e0724a36755aaf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"aa9cf2d4709f42c9eb5dad95a56141738554f1b7d752de0d4b0ca863ed35f08d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"de872c4f9bed78e595aa0309be8b2145ae00fa864cdae74c62b196f02e7a7c68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"e764784e704e2e2b27d861fbb0e9e4e39307c4efc2358883f5a0a0bfaf1ece26","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"ee727effd5d95b42b1f21ecbaf21ff986de2285544c181405cc38338639d85a7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"fdc0e08684b498bbfb00d4bb852da4d3016c65a4ec84f977294ad7bdbf25c4a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"297f54696f0c10752edca7d9cf5c5d2f98707884ddccf8ef2f952e32d9f597c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"299db9852cb6ec237c0f1bb6b3917d59644cef430e83fc2d28e8943682b7e108","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"4bfb160db3532883f03c196af07faf62b876ad474cc0c49e5e501e7fd65537d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"5976bd54ef1912d8a56fae856cd640929ca801e0b38df661fe97065f1e1ddd1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"74c4a0991a9347ceeb2b3ab53adc95cedd09f120cf556321c9b74ae4d3616c90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"7c052020cd8edffb258e72da7eba5bdb0cf97b0d362af401b3cb610f8b906cf6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"8750a839c279cdf29c1a82ccb6189d7fb26d4cb33d0d9e3708556a84f521bd7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"9011219d09cdf07f6d086e4ba313338a0c41037e23acedb902f10a2f46cf7b4c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"9cbcd1fcc44b9dcfeac71a715da83e6c8d7bf9ceee862c10b305cfe44d73e05f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"0d8fad05bfc91d3bf4587ff521f51f71dec38c4bd6167c681739b8c407410634","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"2c5ca9fee600aa40b6dd99a68079eed31e31c0041cca6c9bb81c4e2c28989095","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"556da369df5fa371fae5a268e2663151c01997d0b1338ed3e1e0b97061c14199","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"5a5ab12ceb7567e88b54656617d4a4e3c761b84d7c19fe667549b41a60c4797b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"6d948816532f93e1d1f025d9ed2447096861bf76e34447e5173e22879b9214cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"9ab2b1eb14ea74c42017c9099045c52a9eb9611a341b4d1c5df66b6729fb582b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"a0aef2ff3133e203b8b6be69e707c3391a26d7478459a0d095a7d80d52b7f77e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":100,"anchor_occurrences":6,"after_last_push_count":9,"trend_counts":[0,0,0,0,2,1,6],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":9,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc002b293e0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rx57wpbdxhdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\",\n      \"name\": \"811670f5-5dbe-439c-9697-edae7243b031\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:47:39.993818098Z\",\n      \"endTime\": \"2026-10-03T00:06:42.823230365Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: Creating: virtualmachine creating or updating\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc002b293e0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rx57wpbdxhdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\",\n      \"name\": \"811670f5-5dbe-439c-9697-edae7243b031\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:47:39.993818098Z\",\n      \"endTime\": \"2026-10-03T00:06:42.823230365Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: Creating: virtualmachine creating or updating\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c2b720\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fgzfw6brhrkb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n      \"name\": \"6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:34:15.426377086Z\",\n      \"endTime\": \"2026-10-02T21:53:18.348312497Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: Creating: virtualmachine creating or updating\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000c2b720\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fgzfw6brhrkb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n      \"name\": \"6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:34:15.426377086Z\",\n      \"endTime\": \"2026-10-02T21:53:18.348312497Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: Creating: virtualmachine creating or updating\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00341d8a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8s9pgvgwlgbj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n      \"name\": \"e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:20:19.695876004Z\",\n      \"endTime\": \"2026-10-02T19:39:24.271020967Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: Creating: virtualmachine creating or updating\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc00341d8a0\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8s9pgvgwlgbj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n      \"name\": \"e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:20:19.695876004Z\",\n      \"endTime\": \"2026-10-02T19:39:24.271020967Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: Creating: virtualmachine creating or updating\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"5d19c65c5cc2f83977dfc0bb6da78485985e9a73a9516d73fcb59a9e71c1d6cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"04ae0703900792edf19aad7f285f7ea250c1b64e19621d532ae6fed3c83dba96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"1e6f9fa59cd98bfc284387b5af1ed56122fcb8e577d42d119bd60d5ef8e20605","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"8ae482f04181781883adcf3137eae2256182da0fa9081ef6d8de86debd8ad010","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"8712867601ac02520c8073037f11212160d4494ca873f5f46983b954dae750d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"4dcb3e82c312b27beb2846bcb4e3feac7ef1c6c538ba9c325c9e8ba1cd984500","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":6,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":6,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,0,6],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p76rlg\" to 4.21.0-0.nightly-multi-2026-10-02-021649\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ab88a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-c4g7bpjwkcrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p76rlg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p76rlg\" to 4.21.0-0.nightly-multi-2026-10-02-021649\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000ab88a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-c4g7bpjwkcrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p76rlg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0005e2c30\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9rcfwwnss4kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0005e2c30\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9rcfwwnss4kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00182a270\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7bxvbnqqpfz2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00182a270\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7bxvbnqqpfz2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"1644bbcb2e6380ac66cea640064367161834cb558a92eb398848bad14721b31d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"39ae3bb12ff8df998e369fcb228b237442fff1c7483c6d65c12668d7a03441bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"3c45a70cfc1f78d78efba119b5ce6faa0b80d4c42bcb7dd0ee59de43bc516ac9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"8579cebca7275d6feeb1cb980486c433a32dc36619b5f2acd7985e3b37d192cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"a22a4ddd3b8ff67faf5b163bf25b83cdba837b1c018fd882d7a9b40a6d220ea5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"ec37fd56a0307f63cfac0b7076ce6db1a98af0f908355fdb720d73618014817c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":83.33333333333333,"anchor_occurrences":5,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,1,5],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":4},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001172048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-5sg2h42pm8fv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n      \"name\": \"4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:43:49.738242165Z\",\n      \"endTime\": \"2026-10-02T21:02:49.398233298Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001172048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-5sg2h42pm8fv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n      \"name\": \"4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:43:49.738242165Z\",\n      \"endTime\": \"2026-10-02T21:02:49.398233298Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001522060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nl8vdbhxwtvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n      \"name\": \"31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:27:19.35370587Z\",\n      \"endTime\": \"2026-10-02T18:46:22.521838138Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001522060\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nl8vdbhxwtvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n      \"name\": \"31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T18:27:19.35370587Z\",\n      \"endTime\": \"2026-10-02T18:46:22.521838138Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e761b0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-dwl8n7k5d292\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n      \"name\": \"0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:58:36.615555909Z\",\n      \"endTime\": \"2026-10-02T08:17:38.532193005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e761b0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-dwl8n7k5d292\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n      \"name\": \"0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T07:58:36.615555909Z\",\n      \"endTime\": \"2026-10-02T08:17:38.532193005Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"3ccc455f5000d71ad9adb6c80c8655900d46f92c09932844cfe825c30dea8f29","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"23c88eb451a5cdd79574abe8ef9812935c0e0d4c234696f73be4ca6bce4d6bdc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"d25ebb7feb22db43b9925f83b8d322fdcc4c54a5432fbc9ce5efb6aae6f0e97b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","signature_id":"91fd4fecbd77ed6f997de413f52e960fcad2e588e3fa8f18e505fb83e8c11564","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"fcbce16b0ec7dd896386a692502ebe83bdec088d3b3acd3226882a8945ae4909","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":5,"runs_affected":4,"failed_runs":4,"run_impact_percent":66.66666666666667,"anchor_occurrences":5,"after_last_push_count":42,"trend_counts":[17,1,2,1,10,6,5],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":21,"prior_last_seen_at":"2026-09-27T02:00:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00089caa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b7c2z62bpd6r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\",\n      \"name\": \"0c057ce9-73bf-4556-855a-010dcaa301e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:15:33.73277988Z\",\n      \"endTime\": \"2026-10-02T23:34:33.89328902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-lcxp7-kcfwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-lcxp7-kcfwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00089caa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b7c2z62bpd6r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\",\n      \"name\": \"0c057ce9-73bf-4556-855a-010dcaa301e1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:15:33.73277988Z\",\n      \"endTime\": \"2026-10-02T23:34:33.89328902Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-lcxp7-kcfwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-lcxp7-kcfwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbb780\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-9v5cc2pldl4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\",\n      \"name\": \"64c53475-70e9-4ead-ac9d-b255c14e111e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:37:10.986126021Z\",\n      \"endTime\": \"2026-10-02T20:56:19.371811745Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbb780\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-9v5cc2pldl4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\",\n      \"name\": \"64c53475-70e9-4ead-ac9d-b255c14e111e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T20:37:10.986126021Z\",\n      \"endTime\": \"2026-10-02T20:56:19.371811745Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9c280\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-gg8s9j\" in resourcegroup=\"rg-np-downgrade-gg8s9j-fnkpnt9xd47t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\",\n      \"name\": \"a4186f06-3e4f-4125-8780-405aa920bd3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:29:21.050247186Z\",\n      \"endTime\": \"2026-10-02T19:48:24.533281016Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c9c280\u003e: \n    failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-gg8s9j\" in resourcegroup=\"rg-np-downgrade-gg8s9j-fnkpnt9xd47t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\",\n      \"name\": \"a4186f06-3e4f-4125-8780-405aa920bd3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T19:29:21.050247186Z\",\n      \"endTime\": \"2026-10-02T19:48:24.533281016Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"4c7eda445e675105debe286cdd5590eda5e7186b2f1edc9eaf4ca72af1eec7fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"2b86c91a6a4b689be874849e9832db0f4b59836029e6d1ac55b1fd9ebea622cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"d96bba282a27aaa6fbc92c0663dfd8f01103b1e67a58a200c8d1a2d1b8e4e832","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","signature_id":"f3c82f6a3a610eb2845b45bce009911813c9de13006d2bd6433c177db3846841","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","signature_id":"4bb202a926156d42d840b7a9818cb5b7935514c66bb61eb6995e5c5d73101ec2","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":5,"after_last_push_count":53,"trend_counts":[6,41,1,0,0,0,5],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aae3e0\u003e: \n    failed to create HCP cluster np-dg-minor-t84576: failed waiting for cluster=\"np-dg-minor-t84576\" in resourcegroup=\"rg-np-dg-minor-t84576-xhl45nnpkcxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n      \"name\": \"86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:50:46.033712456Z\",\n      \"endTime\": \"2026-10-03T00:19:52.83457796Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000aae3e0\u003e: \n    failed to create HCP cluster np-dg-minor-t84576: failed waiting for cluster=\"np-dg-minor-t84576\" in resourcegroup=\"rg-np-dg-minor-t84576-xhl45nnpkcxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n      \"name\": \"86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:50:46.033712456Z\",\n      \"endTime\": \"2026-10-03T00:19:52.83457796Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-lg5p6f\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a56dc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-lg5p6f: failed waiting for cluster=\"cp-ystream-upgrade-5-0-lg5p6f\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-b2ldx28lrgzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\",\n      \"name\": \"9c7f2efd-8245-4407-9270-f3937d54358b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:32:39.04806912Z\",\n      \"endTime\": \"2026-10-03T00:01:43.756113041Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-lg5p6f\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a56dc0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-lg5p6f: failed waiting for cluster=\"cp-ystream-upgrade-5-0-lg5p6f\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-b2ldx28lrgzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\",\n      \"name\": \"9c7f2efd-8245-4407-9270-f3937d54358b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:32:39.04806912Z\",\n      \"endTime\": \"2026-10-03T00:01:43.756113041Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-27sjww at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d81140\u003e: \n    failed to create HCP cluster np-skip-minor-27sjww: failed waiting for cluster=\"np-skip-minor-27sjww\" in resourcegroup=\"rg-np-skip-minor-27sjww-kgw6mbppvdmz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n      \"name\": \"74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:50:41.341932888Z\",\n      \"endTime\": \"2026-10-03T00:19:42.833586106Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-27sjww at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d81140\u003e: \n    failed to create HCP cluster np-skip-minor-27sjww: failed waiting for cluster=\"np-skip-minor-27sjww\" in resourcegroup=\"rg-np-skip-minor-27sjww-kgw6mbppvdmz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n      \"name\": \"74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T23:50:41.341932888Z\",\n      \"endTime\": \"2026-10-03T00:19:42.833586106Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"428e779e55c222fac1f48387e160b1242c8a6e675443dd3caec7bdc747ed9c1c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"7eff813a5363b798d17a20887832fb9ba98d961ef78f14f728e98b256371020a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"a70f09283f83907ef445a20f2e96d80236d23d53ed875421b7b5e1da8a6a4b87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"c7e8f4fd3f2bc41bffbc9f2d6790ff6e60abc16e8cbff179fedf0313fca2092e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"f944baaa2c2ba95ce8132743ac5ce22cf1d0b2f92722f2569ba0b7a850637a81","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":4,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a95460\u003e: \n    failed to create NodePool np-k6xcnn: failed waiting for nodepool=\"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\" in resourcegroup=\"rg-candidate-5-1-xcbcjc2xvxnz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n      \"name\": \"f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:05:55.776603892Z\",\n      \"endTime\": \"2026-10-02T21:24:58.213196373Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a95460\u003e: \n    failed to create NodePool np-k6xcnn: failed waiting for nodepool=\"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\" in resourcegroup=\"rg-candidate-5-1-xcbcjc2xvxnz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n      \"name\": \"f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:05:55.776603892Z\",\n      \"endTime\": \"2026-10-02T21:24:58.213196373Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d194a0\u003e: \n    failed to create NodePool np-7lqqwg: failed waiting for nodepool=\"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\" in resourcegroup=\"rg-candidate-4-21-fsj5jdg9tljp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\",\n      \"name\": \"d395f2bf-88ca-4f84-8998-c863978caf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:03:18.868079791Z\",\n      \"endTime\": \"2026-10-02T21:22:18.599909683Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d194a0\u003e: \n    failed to create NodePool np-7lqqwg: failed waiting for nodepool=\"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\" in resourcegroup=\"rg-candidate-4-21-fsj5jdg9tljp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\",\n      \"name\": \"d395f2bf-88ca-4f84-8998-c863978caf3f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T21:03:18.868079791Z\",\n      \"endTime\": \"2026-10-02T21:22:18.599909683Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002d92dc0\u003e: \n    failed to create NodePool np-ljll7c: failed waiting for nodepool=\"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\" in resourcegroup=\"rg-candidate-5-1-m96hr99msv6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\",\n      \"name\": \"3472529a-993b-497a-a853-58314ee41fea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:39:28.8432233Z\",\n      \"endTime\": \"2026-10-02T01:58:35.211672532Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002d92dc0\u003e: \n    failed to create NodePool np-ljll7c: failed waiting for nodepool=\"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\" in resourcegroup=\"rg-candidate-5-1-m96hr99msv6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\",\n      \"name\": \"3472529a-993b-497a-a853-58314ee41fea\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T01:39:28.8432233Z\",\n      \"endTime\": \"2026-10-02T01:58:35.211672532Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"4b3072ea271a5fa9392b9001a31d9bdf8953c887295f5949247942b3ff4ae13e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","signature_id":"7ed334e95fee4ff7d071b052175951829e19af47fe9ec8e38db681d9e969f735","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"02f2d68ee929abf0d14dbf066cea47b45e31fa5a7861dd29f5ff1ae2dda13550","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","signature_id":"2e01f42474260e70f5cc8017801f4c45f4df9dbc895992ce253206654bfb0587","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":32,"trend_counts":[5,0,5,5,0,0,2],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001680330\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kcc2h87qnl8v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001680330\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kcc2h87qnl8v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002910ff0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-rbgmrr9dbzzr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002910ff0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-rbgmrr9dbzzr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"487336fbee3aa09886b1644e07daab686d89b729e7e6fdb636dc41d4196e0756","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"a6951ac14ce4de69f00809a996ba16b47c97b931a257020baf028934f05e2b52","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":28,"trend_counts":[2,1,4,4,4,0,2],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001214a20\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r44ftv\" in resourcegroup=\"rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r44ftv/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001214a20\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r44ftv\" in resourcegroup=\"rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r44ftv/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be25e0\u003e: \n    failed to create NodePool np-f6fpxn: failed starting nodepool creation \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\" in resourcegroup=\"rg-nightly-4-20-9z2m2dsr8b8d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9z2m2dsr8b8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-f6fpxn/nodePools/np-f6fpxn\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be25e0\u003e: \n    failed to create NodePool np-f6fpxn: failed starting nodepool creation \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\" in resourcegroup=\"rg-nightly-4-20-9z2m2dsr8b8d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9z2m2dsr8b8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-f6fpxn/nodePools/np-f6fpxn\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"088d80b26ce242a6a23ce42eb36d87da5a2ba3b0cb09615c4e35d5a5873f05c2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"aae832d22fd2e270ae046b4cd1453e86b9c22535700ac5df9912205534e9a89c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"02985e80f9e6dd23af129635b4bd016a473a6a57b067e6f88bbc87e7f20a5595","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-26..2026-10-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00141a900\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-kmfrpjr862hw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kmfrpjr862hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00141a900\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-kmfrpjr862hw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kmfrpjr862hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a36c0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-tsckb2km58j7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-tsckb2km58j7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010a36c0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-tsckb2km58j7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-tsckb2km58j7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"770ee469d83ba1d5d599e53e42dfd2f872e81b78bafe998012dfc850626289cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","signature_id":"fc18079b3e233421ba88f574156106a7974d4521c46097383314a9ad391da2d6","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}