{"meta":{"start_date":"2026-10-01","end_date":"2026-10-01","start_at":"2026-10-01T00:00:00Z","end_at":"2026-10-02T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T10:00:05Z","environments":["stg"]},"environments":[{"environment":"stg","summary":{"total_runs":4,"failed_runs":4,"raw_occurrences":28,"matched_occurrences":27,"runs_affected":3},"rows":[{"environment":"stg","failure_pattern_id":"529e53d993c6d6fde7fea2440ec7300376970a84937f06a76bbb493596a4d7b6","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":10,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,0,10],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-kbwzzs\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00104fe60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb6v87bszxhb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-kbwzzs\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-kbwzzs\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00104fe60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb6v87bszxhb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-kbwzzs\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-hwc6dp\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00164d620\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-txlxhwglc5dq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-hwc6dp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-hwc6dp\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00164d620\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-txlxhwglc5dq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-hwc6dp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-sq7dhp\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015c4e70\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-vjpbzl2jbr62/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-sq7dhp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-sq7dhp\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015c4e70\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-vjpbzl2jbr62/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-sq7dhp\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"11a7f777efd16a4602f165155402df79554a160ce761b48987b2fa81eaa853ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"1719e4976d13747a5ba884e6917c1488f67a8a955b14f27daff4e13ed2902037","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"2388c2f6af983771920940504bcf433d725fb8c7cc1e81c55f6ab467636ed7ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"5565e5f16c645b130d162f8ebf645832f59d83ad60615990833bdafaca4b6ef8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"844c1e8cf0cc01d92e68b7c7011864371b606ef8f2d55622248024c05d7ce83e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"a09d9f975ede859b703696d2a88baac07d837fd9c31b0ebe46690181381da93a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"a8eb2d795a11c6dd632258debe4cb2cff59cae61bb73a703f5d0ea485effd6ec","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"c14433cba7a1a57c209cdd5665919f16d75f3d85ba2502e0034b92b8f4aee701","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"f03158936394395704c5b6ff64b4370dd9e2548b869de7e69671c413ff75621f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"fd3be8a2cabe6e92eed1d71ec31a986980accf6d34064931d1b31fd3558ce497","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":8,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":8,"after_last_push_count":8,"trend_counts":[0,0,0,0,0,0,8],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009252c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-tfpfzw4vzd4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\",\n      \"name\": \"8e650b8c-c903-48db-ae29-20e1c0f74604\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:01:41.16595227Z\",\n      \"endTime\": \"2026-10-01T23:20:45.151714689Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: Creating: virtualmachine creating or updating\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009252c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-tfpfzw4vzd4n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8e650b8c-c903-48db-ae29-20e1c0f74604\",\n      \"name\": \"8e650b8c-c903-48db-ae29-20e1c0f74604\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:01:41.16595227Z\",\n      \"endTime\": \"2026-10-01T23:20:45.151714689Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-8nqmx: Creating: virtualmachine creating or updating\\nMachine w8n1y8f9d9k3r1o-np-1-mllsn-dpzt8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001f200c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4rdzlcbp8jqd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\",\n      \"name\": \"7941e9fc-eabb-4912-b47f-1619c832d931\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:02:15.574287096Z\",\n      \"endTime\": \"2026-10-01T23:21:15.149755975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001f200c0\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: rate: Wait(n=1) would exceed context deadline\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4rdzlcbp8jqd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7941e9fc-eabb-4912-b47f-1619c832d931\",\n      \"name\": \"7941e9fc-eabb-4912-b47f-1619c832d931\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:02:15.574287096Z\",\n      \"endTime\": \"2026-10-01T23:21:15.149755975Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-tsk7z-g2d4q: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-tsk7z-rcxdh: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012e40\u003e: \n    failed to create NodePool np-6g6tzd: failed waiting for nodepool=\"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\" in resourcegroup=\"rg-candidate-4-22-xz7x2977wmz2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n      \"name\": \"3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:22:22.461339415Z\",\n      \"endTime\": \"2026-10-01T23:41:27.247269633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: Creating: virtualmachine creating or updating\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001012e40\u003e: \n    failed to create NodePool np-6g6tzd: failed waiting for nodepool=\"np-6g6tzd\" for cluster \"cluster-candidate-4-22-6g6tzd\" in resourcegroup=\"rg-candidate-4-22-xz7x2977wmz2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n      \"name\": \"3c484f73-f02b-4ece-b374-6bbafdd0e00e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:22:22.461339415Z\",\n      \"endTime\": \"2026-10-01T23:41:27.247269633Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-564bh: Creating: virtualmachine creating or updating\\nMachine m6u9b8d4j0a9a8z-np-6g6tzd-g5w62-777td: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"1b39a1fcea6ce322dd8fa1bd054343dbd452f0f0491cacc4f8b9ef7b6a0493be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"2169238a3efc65d2f002e87d4107ee839610a68181a3b21a5f5f8555c8c0152b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"261d2ba977979da2043d1d05e7fda09b0050628d435b4e8f6c70c65d7ede0f72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"4247285a4a7e0ce74bb11c7155b281ec95aa3b9b5e58b9fb3220d4d61dc786b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"577a2f2cdd3de8213f2886c7670e9678f9d789b1a765ea47d5053e8926ce155f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"84ad30e1bfed5ae9658a5da88f0ca1af31ae55f3c693883c462c46dcf7608e01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"b4159087ef39a5abb0c42cbb58f2ea45413e0c9d5e74cdc7078f52b05a7b45c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"bdfd444f6b247ef48624113640b671ccd50695d691f9d049b710ad7273122875","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":75,"anchor_occurrences":6,"after_last_push_count":37,"trend_counts":[0,17,1,2,1,10,6],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-27T02:00:24Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f50a20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-6svllmjvpbpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\",\n      \"name\": \"7c5e553a-2bca-431a-819c-94122ad78b6b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:04:50.703195637Z\",\n      \"endTime\": \"2026-10-02T00:23:53.181881109Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f50a20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-6svllmjvpbpf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7c5e553a-2bca-431a-819c-94122ad78b6b\",\n      \"name\": \"7c5e553a-2bca-431a-819c-94122ad78b6b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:04:50.703195637Z\",\n      \"endTime\": \"2026-10-02T00:23:53.181881109Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-6svllmjvpbpf--managed/providers/Microsoft.Compute/virtualMachines/i6r6j5b5y3t4r7g-np-1-tzs6p-6v7dr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a16440\u003e: \n    failed to create NodePool np-k96n75: failed waiting for nodepool=\"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\" in resourcegroup=\"rg-candidate-5-0-gclccxznszk7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n      \"name\": \"e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:23:36.842890739Z\",\n      \"endTime\": \"2026-10-01T23:42:37.244899482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: Creating: virtualmachine creating or updating\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a16440\u003e: \n    failed to create NodePool np-k96n75: failed waiting for nodepool=\"np-k96n75\" for cluster \"cluster-candidate-5-0-k96n75\" in resourcegroup=\"rg-candidate-5-0-gclccxznszk7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n      \"name\": \"e0f4877f-a7bc-408a-891f-bd985b38bfd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:23:36.842890739Z\",\n      \"endTime\": \"2026-10-01T23:42:37.244899482Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-6t7jp: Creating: virtualmachine creating or updating\\nMachine l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-gclccxznszk7-candidate-k96n75--managed/providers/Microsoft.Compute/virtualMachines/l0o8x0e0c2l1i0q-np-k96n75-6284m-gt7km to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00081f300\u003e: \n    failed to create NodePool np-8l66wr: failed waiting for nodepool=\"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\" in resourcegroup=\"rg-candidate-4-20-66r4zk7wm6rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\",\n      \"name\": \"1513980f-1a90-4c5b-865f-c0a3b6601404\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T03:20:14.931117191Z\",\n      \"endTime\": \"2026-10-01T03:39:22.388104494Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00081f300\u003e: \n    failed to create NodePool np-8l66wr: failed waiting for nodepool=\"np-8l66wr\" for cluster \"cluster-candidate-4-20-8l66wr\" in resourcegroup=\"rg-candidate-4-20-66r4zk7wm6rk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1513980f-1a90-4c5b-865f-c0a3b6601404\",\n      \"name\": \"1513980f-1a90-4c5b-865f-c0a3b6601404\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T03:20:14.931117191Z\",\n      \"endTime\": \"2026-10-01T03:39:22.388104494Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-66r4zk7wm6rk-candidate-8l66wr--managed/providers/Microsoft.Compute/virtualMachines/y7w9c5q7k5c5l6z-np-8l66wr-wzjtv-cnmkc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"153a3254e17592610cd68e651bfb6cbf2394189d89044802b8c3b533e149203a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"6fc83d6684638cb567013759a7193f362c34949e41b3a8668c62f64a251d057c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"623dfd830d068e8fc297a04ee4a56d7cb6e4821c0251af7874f2b77ba7c26fa6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"c574b07b3114ca80e839d7f5d60517775ee576de2d943538a90489d57cb3eec5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105477920788058112","occurred_at":"2026-10-01T02:00:28Z","signature_id":"f47c5337335a014c2bed2f388006a403bbb00ccd35a477f73d5472519ff75085","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105450319994425344","occurred_at":"2026-10-01T00:10:48Z","signature_id":"b47c99754a6a1f045aa9bf003dd005e8d53c7b6f39de576acac3bb76b9fadb06","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c16198\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-mrv84pkzs8gp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n      \"name\": \"c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:02:38.499943334Z\",\n      \"endTime\": \"2026-10-01T23:21:45.00235976Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001c16198\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-mrv84pkzs8gp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n      \"name\": \"c363e8fc-7ec7-47fd-882a-c1fa371683da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:02:38.499943334Z\",\n      \"endTime\": \"2026-10-01T23:21:45.00235976Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-rjv4k-5xzgh: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-rjv4k-7mcss: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"951b0a5c5a70e3b35bb3f64267eb4dc0e2b6817b2864d1a40fa29654b0ff2a81","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"a146c4d50a72c43e4a69c21e18547ba5a2a0a2b53d6985079637bcec0f5acdda","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019901c8\u003e: \n    resource group \"oidc-wi-nq9xg56z8ptc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nq9xg56z8ptc\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n      \"name\": \"b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:11:10.101784102Z\",\n      \"endTime\": \"2026-10-02T00:35:13.452817702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t6a9r7piu3mrhnemk9adlcvlla5mb7v still exists (deletion dispatched at 2026-10-02T00:11:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0019901c8\u003e: \n    resource group \"oidc-wi-nq9xg56z8ptc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-nq9xg56z8ptc\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n      \"name\": \"b2411afb-26dd-4be7-a31a-e99647e9b52e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:11:10.101784102Z\",\n      \"endTime\": \"2026-10-02T00:35:13.452817702Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t6a9r7piu3mrhnemk9adlcvlla5mb7v still exists (deletion dispatched at 2026-10-02T00:11:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"307bc38fc0da1536f942492cf74c8e559878301b0fc5629d37255c20e9ee7ec4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-g79pk8vm4jx6\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000d82400\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-g79pk8vm4jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n      \"name\": \"d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:53:42.212331844Z\",\n      \"endTime\": \"2026-10-02T00:12:43.153935456Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: Creating: virtualmachine creating or updating\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000d82400\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-g79pk8vm4jx6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n      \"name\": \"d7afe39e-2daf-4d6c-a78a-cef81fb1f217\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:53:42.212331844Z\",\n      \"endTime\": \"2026-10-02T00:12:43.153935456Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-c6ctv: Creating: virtualmachine creating or updating\\nMachine i1n3h3r5l8p6b0o-np-delete-main-qzd58-qt8wr: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105784694955053056","occurred_at":"2026-10-01T22:19:29Z","signature_id":"bff4c5f332cfa283a8daa21713459950af8584404733577995b07a2a2bfcc5a8","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}