{"meta":{"date":"2026-10-02","timezone":"UTC","generated_at":"2026-10-14T11:56:06Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":66,"failed_runs":48,"runs_with_occurrences":48,"runs_with_failure_pattern_matches":46,"runs_with_unmatched_occurrences":2,"runs_without_occurrence_rows":18,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106171709915664384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7326,"pr_state":"closed","pr_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","final_merged_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T23:57:21Z","started_at":"2026-10-02T23:57:21Z","completed_at":"2026-10-03T02:43:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T03:00:48.993279418Z","timing_metadata_checked_at":"2026-10-03T03:00:48.993279418Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T03:00:48.996533308Z","region_metadata_checked_at":"2026-10-03T03:00:48.996533308Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"48d94e043e592202b35671468b8904845f0b97fe18b18687e18563d54561708f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106171709915664384","occurred_at":"2026-10-02T23:57:21Z","failed_at":"e2e","signature_id":"0c35eab3a95f3ff5276180583f68218d788a369962226e679eb22ef600b7e783","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x152c481045e0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s55djgjncgbr\" to finish creating: GET https://rp.j5664384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0x152c481045e0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s55djgjncgbr\" to finish creating: GET https://rp.j5664384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bd7ec69c-0275-4790-bd33-ff2503c1f27a\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}},{"row_id":"533accb7c572297eeaa5998adfaae2cb9a5fa962540f5448cff1c28ab6a8ec01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106171709915664384","occurred_at":"2026-10-02T23:57:21Z","failed_at":"e2e","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106164805017538560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7310,"pr_state":"closed","pr_sha":"03087d68ce4c9cae7c9c78fc4442cfd50af8ce47","final_merged_sha":"30ad98fd4402fbef6733091819c97a3b7da933b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T23:29:54Z","started_at":"2026-10-02T23:29:54Z","completed_at":"2026-10-03T02:01:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T02:18:30.239171362Z","timing_metadata_checked_at":"2026-10-03T02:18:30.239171362Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T02:18:30.242344352Z","region_metadata_checked_at":"2026-10-03T02:18:30.242344352Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b6fc9adea415f98ab28b159aa2872d87ec13bff4a82effa79e0b0e15fc5d8fbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7310/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106164805017538560","occurred_at":"2026-10-02T23:29:54Z","failed_at":"e2e","signature_id":"1d1306e065199e57cf378e8b1110f97b0d8741e76dab448d010dc8c6539462a1","test_name":"SRE should be able to perform alert processing rule CRUD operations","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:493]: failed to create alert processing rule\nUnexpected error:\n \u003c*errors.errorString | 0xc0013d1ed0\u003e: \n expected status 200, got 400: {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid scope(s) in request: at least one of these scopes is required: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7538560/providers/Microsoft.Monitor/accounts/hcps-j7538560,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7538560/providers/Microsoft.Monitor/accounts/services-j7538560\"\n }\n }\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:493]: failed to create alert processing rule\nUnexpected error:\n \u003c*errors.errorString | 0xc0013d1ed0\u003e: \n expected status 200, got 400: {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid scope(s) in request: at least one of these scopes is required: /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7538560/providers/Microsoft.Monitor/accounts/hcps-j7538560,/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7538560/providers/Microsoft.Monitor/accounts/services-j7538560\"\n }\n }\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc37414a0bb07deceea16e285252404618c039575a3001bedcbcbd958dd8e87d","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid scope(s) in request: at least one of these scopes is required: /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Monitor/accounts/hcps-j7538560,/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Monitor/accounts/services-j7538560; provider Microsoft.Monitor","search_query":"\"error\": {"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["cc37414a0bb07deceea16e285252404618c039575a3001bedcbcbd958dd8e87d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157602076364800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"open","pr_sha":"dae2fd713c930dcbfd480d53f1bc49cc194723d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T23:01:13Z","started_at":"2026-10-02T23:01:13Z","completed_at":"2026-10-03T01:27:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T01:36:15.039952655Z","timing_metadata_checked_at":"2026-10-03T01:36:15.039952655Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T01:36:15.044015141Z","region_metadata_checked_at":"2026-10-03T01:36:15.044015141Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2cf3c7c028d051e6fa7145418615d07b5e1702e6420e953ec68028669941fdb6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157602076364800","occurred_at":"2026-10-02T23:01:13Z","failed_at":"e2e","signature_id":"34de2ab11f5adbf068c342e2d31a9ac77891b0e38b007bfdbb6d9da4bc98f473","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-sd9hfz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001182600\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-sd9hfz\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt\" to finish creating: GET https://rp.j6364800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6ca047c0-02df-4aac-a493-0a13f044d7dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6ca047c0-02df-4aac-a493-0a13f044d7dc\",\n \"name\": \"6ca047c0-02df-4aac-a493-0a13f044d7dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:01:24.493949107Z\",\n \"endTime\": \"2026-10-03T00:20:29.737570785Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p to exist\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p to exist\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-sd9hfz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001182600\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-5-0-sd9hfz\" in resourcegroup=\"rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt\" to finish creating: GET https://rp.j6364800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6ca047c0-02df-4aac-a493-0a13f044d7dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6ca047c0-02df-4aac-a493-0a13f044d7dc\",\n \"name\": \"6ca047c0-02df-4aac-a493-0a13f044d7dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:01:24.493949107Z\",\n \"endTime\": \"2026-10-03T00:20:29.737570785Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p to exist\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-dgc2p to exist\\nMachine m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-5-0-cmdz5pmslwlt-cp-ystream-den-95628066/providers/Microsoft.Compute/virtualMachines/m5o4v4c8c8p6v5f-np-deny-pods-wc6td-tljpj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"808d565e463b0b437e5cdb606998870b74652995b0e414ff0c6431232849b7ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157602076364800","occurred_at":"2026-10-02T23:01:13Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T00:06:25Z\n Ended: 2026-10-03T00:13:29Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6364800-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157368797564928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7331,"pr_state":"closed","pr_sha":"53a4ed21d7b546b4f4ac2fb406d7d9bec086289b","final_merged_sha":"d5ee24cc09296f5a16b33a9dbe4bd808f7e62ec2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T23:00:21Z","started_at":"2026-10-02T23:00:21Z","completed_at":"2026-10-03T01:23:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T01:36:15.783526668Z","timing_metadata_checked_at":"2026-10-03T01:36:15.783526668Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T01:36:15.786379158Z","region_metadata_checked_at":"2026-10-03T01:36:15.786379158Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"aa97078d1b6ea72c23eb6cc7cb1dba7da854d59ce3e41da758be729bfa326fcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157368797564928","occurred_at":"2026-10-02T23:00:21Z","failed_at":"alert","signature_id":"04501c7bb397428ef39e831a7c967076ecd521dff084b9e5def70ae9b291e02a","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T00:07:24Z\n Ended: 2026-10-03T00:19:25Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7564928-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-03T00:08:27Z\n Ended: 2026-10-03T00:27:27Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7564928-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"c0c03457bf761fd77c1093b8c69e409e027e295b11b9fd0f470e77e81cccb058","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7331/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157368797564928","occurred_at":"2026-10-02T23:00:21Z","failed_at":"alert","signature_id":"fa3c4dd358a0d346db704e5b27ea7256c0e149b36b853364eed04a48b1ad2821","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 9 time(s) (1 unknown, 8 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T00:22:54Z\n Ended: 2026-10-03T00:28:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7564928-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6vba5aukt21s79ka316m1hqn628hho-s4a5z1v1k7m8i6h\", pod=\"openshift-apiserver-64ddcdd5c4-tjv4j\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6vba5aukt21s79ka316m1hqn628hho-s4a5z1v1k7m8i6h/openshift-apiserver-64ddcdd5c4-tjv4j has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7330/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157312019271680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7330,"pr_state":"open","pr_sha":"ca9f03851b92e408e22caac53a6d8c02b986da7f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T23:00:08Z","started_at":"2026-10-02T23:00:08Z","completed_at":"2026-10-03T01:28:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T01:36:15.394544874Z","timing_metadata_checked_at":"2026-10-03T01:36:15.394544874Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T01:36:15.399748955Z","region_metadata_checked_at":"2026-10-03T01:36:15.399748955Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f4e3c449d50982684da3bbed44d9fee26bd83d75087e6e892e647dc811468435","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7330/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106157312019271680","occurred_at":"2026-10-02T23:00:08Z","failed_at":"alert","signature_id":"46d148353c0b164c86fe30292f88ae89ac37e04f171a8c6e854febc1e90da782","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-03T00:19:29Z\n Ended: 2026-10-03T00:25:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9271680-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9271680-svc).\nFiring 2:\n State: Resolved\n Started: 2026-10-03T00:35:30Z\n Ended: 2026-10-03T00:41:31Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j9271680-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j9271680-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106156228282421248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7326,"pr_state":"closed","pr_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","final_merged_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T22:55:49Z","started_at":"2026-10-02T22:55:49Z","completed_at":"2026-10-02T23:23:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T23:40:47.153036523Z","timing_metadata_checked_at":"2026-10-02T23:40:47.153036523Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-02T23:40:47.156359111Z","region_metadata_checked_at":"2026-10-03T00:43:39.16712661Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"99dc3db71d26b00a3b4695b92524d138d7c24f95fe8d8d2b1cdac724405b77bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106156228282421248","occurred_at":"2026-10-02T22:55:49Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7313,"pr_state":"open","pr_sha":"c163c5279eda212ed7a42f1173d857f29bca3ec3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T22:03:59Z","started_at":"2026-10-02T22:03:59Z","completed_at":"2026-10-03T00:18:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T00:32:58.075334672Z","timing_metadata_checked_at":"2026-10-03T00:32:58.075334672Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T00:32:58.078920259Z","region_metadata_checked_at":"2026-10-03T00:32:58.078920259Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"10043a320e3848cfa191e2aefb177d96d87cd43c0696d1da78bedbcc1e48aad4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","failed_at":"e2e","signature_id":"affbacfd51e93098bd3964ec613491d9c1e15b85104d19de75d8c835e704fdce","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-5fqc8n\" on minor version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b34440\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-22-5fqc8n\" in resourcegroup=\"rg-zstream-upgrade-4-22-tqtxbbsf75hl\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b62d2f-e329-486f-a6aa-a4fc7c209017\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b62d2f-e329-486f-a6aa-a4fc7c209017\",\n \"name\": \"30b62d2f-e329-486f-a6aa-a4fc7c209017\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:01:14.974688957Z\",\n \"endTime\": \"2026-10-02T23:45:16.632504331Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-5fqc8n\" on minor version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b34440\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-22-5fqc8n\" in resourcegroup=\"rg-zstream-upgrade-4-22-tqtxbbsf75hl\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b62d2f-e329-486f-a6aa-a4fc7c209017\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/30b62d2f-e329-486f-a6aa-a4fc7c209017\",\n \"name\": \"30b62d2f-e329-486f-a6aa-a4fc7c209017\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:01:14.974688957Z\",\n \"endTime\": \"2026-10-02T23:45:16.632504331Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7701037f3d8ca311cabf690beb6ae29c6d7792b9a25fa5faa4e0ec0d85bda944","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","failed_at":"e2e","signature_id":"0660e5aeb2a877fab962d22ed49cccd7985f44881bfda87726aaa352b96da22a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-fskm94\" on minor version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a32b40\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-20-fskm94\" in resourcegroup=\"rg-zstream-upgrade-4-20-7d7sxcc5nbp4\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a14b50b-454e-42ea-8dd4-4340cfdff6fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a14b50b-454e-42ea-8dd4-4340cfdff6fa\",\n \"name\": \"0a14b50b-454e-42ea-8dd4-4340cfdff6fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:59:38.87341489Z\",\n \"endTime\": \"2026-10-02T23:43:46.612931387Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-fskm94\" on minor version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a32b40\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-20-fskm94\" in resourcegroup=\"rg-zstream-upgrade-4-20-7d7sxcc5nbp4\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a14b50b-454e-42ea-8dd4-4340cfdff6fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a14b50b-454e-42ea-8dd4-4340cfdff6fa\",\n \"name\": \"0a14b50b-454e-42ea-8dd4-4340cfdff6fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:59:38.87341489Z\",\n \"endTime\": \"2026-10-02T23:43:46.612931387Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8ff82288ea39776fb3280414085cac17eb3034fe9aa12b25532d3855e3eb51d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","failed_at":"e2e","signature_id":"b7678a0e4b4e24474229f76609c785b6fb93d2d584801f6c25d6ac0623415d59","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-xxdkbk\" on minor version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060c400\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-21-xxdkbk\" in resourcegroup=\"rg-zstream-upgrade-4-21-fkvp7fgld7fx\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e466c8d-b439-4e8d-9c16-270890d65866\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e466c8d-b439-4e8d-9c16-270890d65866\",\n \"name\": \"3e466c8d-b439-4e8d-9c16-270890d65866\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:59:19.732496702Z\",\n \"endTime\": \"2026-10-02T23:43:26.633598885Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-xxdkbk\" on minor version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060c400\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-21-xxdkbk\" in resourcegroup=\"rg-zstream-upgrade-4-21-fkvp7fgld7fx\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e466c8d-b439-4e8d-9c16-270890d65866\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e466c8d-b439-4e8d-9c16-270890d65866\",\n \"name\": \"3e466c8d-b439-4e8d-9c16-270890d65866\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:59:19.732496702Z\",\n \"endTime\": \"2026-10-02T23:43:26.633598885Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a69ac459cf157eb8360fb7af54749ea025ec54f461089b71430239cef364d8bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","failed_at":"e2e","signature_id":"5bd9a565d39a5a143cbf016e6eacabe5fc41e6b584fd89893528ebc43bc9173f","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6f800\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xzgkv9b984lz\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc6e51ed-79e2-431d-8b60-c9c38f639a11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc6e51ed-79e2-431d-8b60-c9c38f639a11\",\n \"name\": \"bc6e51ed-79e2-431d-8b60-c9c38f639a11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:58:51.980435313Z\",\n \"endTime\": \"2026-10-02T23:08:01.293538008Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6f800\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xzgkv9b984lz\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc6e51ed-79e2-431d-8b60-c9c38f639a11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc6e51ed-79e2-431d-8b60-c9c38f639a11\",\n \"name\": \"bc6e51ed-79e2-431d-8b60-c9c38f639a11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:58:51.980435313Z\",\n \"endTime\": \"2026-10-02T23:08:01.293538008Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bbafa9780fe83daf26f55ee70ad0cb05de1b5db68e68f0098db23ba2bc7d7bc4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","failed_at":"e2e","signature_id":"ca2142bbb3700266ca07c7e4c24330f499e20b7321800d8b3c755ba6c4b16e28","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-qrrxbq\" on minor version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7e340\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-5-0-qrrxbq\" in resourcegroup=\"rg-zstream-upgrade-5-0-bnn4f57kv77n\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7191531-446c-4865-aac9-dd2569450626\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7191531-446c-4865-aac9-dd2569450626\",\n \"name\": \"c7191531-446c-4865-aac9-dd2569450626\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:58:13.4607593Z\",\n \"endTime\": \"2026-10-02T23:42:16.686387844Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-qrrxbq\" on minor version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7e340\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-5-0-qrrxbq\" in resourcegroup=\"rg-zstream-upgrade-5-0-bnn4f57kv77n\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7191531-446c-4865-aac9-dd2569450626\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c7191531-446c-4865-aac9-dd2569450626\",\n \"name\": \"c7191531-446c-4865-aac9-dd2569450626\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T22:58:13.4607593Z\",\n \"endTime\": \"2026-10-02T23:42:16.686387844Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d719888b41ec958ea34b34df3a967db2847d9d06ad5d31727f5451cc621c09d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7313/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106143183632076800","occurred_at":"2026-10-02T22:03:59Z","failed_at":"e2e","signature_id":"4d059280747339409745ae35874bcfc7c6f8797f43ab108053ff4751fb5cfa9b","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-59v6sn\" on minor version 4.23\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be1060\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-23-59v6sn\" in resourcegroup=\"rg-zstream-upgrade-4-23-dwdrgs9rpkt4\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2c90638-d31c-4819-aa00-1f73910cfe32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2c90638-d31c-4819-aa00-1f73910cfe32\",\n \"name\": \"f2c90638-d31c-4819-aa00-1f73910cfe32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:00:29.412121269Z\",\n \"endTime\": \"2026-10-02T23:44:36.589759019Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-59v6sn\" on minor version 4.23\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be1060\u003e: \n failed waiting for hcpCluster=\"cluster-zstream-4-23-59v6sn\" in resourcegroup=\"rg-zstream-upgrade-4-23-dwdrgs9rpkt4\" to finish updating: GET https://rp.j2076800.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2c90638-d31c-4819-aa00-1f73910cfe32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f2c90638-d31c-4819-aa00-1f73910cfe32\",\n \"name\": \"f2c90638-d31c-4819-aa00-1f73910cfe32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:00:29.412121269Z\",\n \"endTime\": \"2026-10-02T23:44:36.589759019Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \\\"hypershift.openshift.io/cluster-size-override\\\" is \\\"e2e_minimal\\\", want unset\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster update did not complete before the deadline; [hypershiftHostedCluster] hypershift HostedCluster annotation \"hypershift.openshift.io/cluster-size-override\" is \"e2e_minimal; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["56d972e76712743c510249b7bfaf8d1cef7f97d62d33d6258594d264ca2ff6b2"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106141674764767232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7321,"pr_state":"closed","pr_sha":"c2eb4d5a17ddbf56ef3a8d378d7990dee66ec717","final_merged_sha":"c2eb4d5a17ddbf56ef3a8d378d7990dee66ec717","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T21:58:00Z","started_at":"2026-10-02T21:58:00Z","completed_at":"2026-10-03T00:24:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T00:32:58.445180013Z","timing_metadata_checked_at":"2026-10-03T00:32:58.445180013Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T00:32:58.448109602Z","region_metadata_checked_at":"2026-10-03T00:32:58.448109602Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106141275559301120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7278,"pr_state":"open","pr_sha":"71b3b079cedc017da41f73f2556116e3e6b944b8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T21:56:24Z","started_at":"2026-10-02T21:56:24Z","completed_at":"2026-10-03T00:15:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T00:22:55.754043634Z","timing_metadata_checked_at":"2026-10-03T00:22:55.754043634Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T00:22:55.756720826Z","region_metadata_checked_at":"2026-10-03T00:22:55.756720826Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"346c803c0a499be58d622de648587f147c2feef67f89d7c6c593ffc58c942f95","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106141275559301120","occurred_at":"2026-10-02T21:56:24Z","failed_at":"e2e","signature_id":"63d927cd1c7a9670c7278fe12e140bf43f44a0266f7ac91ced09dcf4f09830bc","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad2a60\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-9h4twv\" in resourcegroup=\"rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx\" to finish creating: GET https://rp.j9301120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/798eb1c5-8054-497c-b466-6bdb523be072\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/798eb1c5-8054-497c-b466-6bdb523be072\",\n \"name\": \"798eb1c5-8054-497c-b466-6bdb523be072\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:16:07.311984652Z\",\n \"endTime\": \"2026-10-02T23:35:13.399526187Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx-np-upgrade-9h-7e5530d9/providers/Microsoft.Compute/virtualMachines/p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx-np-upgrade-9h-7e5530d9/providers/Microsoft.Compute/virtualMachines/p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad2a60\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-9h4twv\" in resourcegroup=\"rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx\" to finish creating: GET https://rp.j9301120.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/798eb1c5-8054-497c-b466-6bdb523be072\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/798eb1c5-8054-497c-b466-6bdb523be072\",\n \"name\": \"798eb1c5-8054-497c-b466-6bdb523be072\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:16:07.311984652Z\",\n \"endTime\": \"2026-10-02T23:35:13.399526187Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx-np-upgrade-9h-7e5530d9/providers/Microsoft.Compute/virtualMachines/p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9h4twv-s2qpdtw4l5dx-np-upgrade-9h-7e5530d9/providers/Microsoft.Compute/virtualMachines/p5q9c9u0b2w8w7g-npupgrade-4-20-2vngn-hhgjs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a77695906571df16780e9a53b14fe4358ee154362cb3aa511b42e9b41656679f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7278/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106141275559301120","occurred_at":"2026-10-02T21:56:24Z","failed_at":"e2e","signature_id":"7060bb637cf275cb6e0005a64cdfdea3e58f04131055a1a620001107ea86bc5a","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc2640\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bc2640\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.003s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7088/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106138605758976000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7088,"pr_state":"open","pr_sha":"e247e32992f72af12f8457e6e4ae8b063b66ff91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T21:45:45Z","started_at":"2026-10-02T21:45:45Z","completed_at":"2026-10-03T00:15:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T00:22:55.607340702Z","timing_metadata_checked_at":"2026-10-03T00:22:55.607340702Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T00:22:55.610756091Z","region_metadata_checked_at":"2026-10-03T00:22:55.610756091Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106138142363881472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7305,"pr_state":"open","pr_sha":"70c38907e0203b243b89ca27c2ca9543161d8720","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T21:43:57Z","started_at":"2026-10-02T21:43:57Z","completed_at":"2026-10-03T00:27:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T00:43:39.068996718Z","timing_metadata_checked_at":"2026-10-03T00:43:39.068996718Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T00:43:39.072979205Z","region_metadata_checked_at":"2026-10-03T00:43:39.072979205Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106117061750034432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"closed","pr_sha":"7da218ee42bad349e24419fd993865ad4b1fe8ad","final_merged_sha":"ea2df83db19482a983a29965803a71b1f8158ab7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T20:20:11Z","started_at":"2026-10-02T20:20:11Z","completed_at":"2026-10-02T23:03:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T23:19:59.491910353Z","timing_metadata_checked_at":"2026-10-02T23:19:59.491910353Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T23:19:59.495538842Z","region_metadata_checked_at":"2026-10-02T23:19:59.495538842Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"58b2319d8042f67d09cad8eff95c6bb5a95f558f619588e1759d08cc90a65f94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106117061750034432","occurred_at":"2026-10-02T20:20:11Z","failed_at":"alert","signature_id":"75a9926b17ec082959d8c07a3aca44e4b153424b74c91ed1f5a5f6cbaf33d33f","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T22:50:00Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0034432-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.34:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0034432/providers/microsoft.monitor/accounts/services-j0034432\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-fdbccb774-c7nq9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j0034432-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-02T22:50:00Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j0034432-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.34:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0034432/providers/microsoft.monitor/accounts/services-j0034432\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-fdbccb774-c7nq9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j0034432-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}},{"row_id":"5cdfb102e228f7533b6cda1a3d330b278133bc6fa179cb663487479278fa9031","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106117061750034432","occurred_at":"2026-10-02T20:20:11Z","failed_at":"e2e","signature_id":"ce7070d24262cdcf05d41676703c04f5be7f690258e7e4c7e074539c5acf5075","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008d1b20\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-b86qkn\" in resourcegroup=\"rg-np-version-upgrade-b86qkn-wdjflpkjvp8p\" to finish creating: GET https://rp.j0034432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba8abfad-4308-4775-a81e-7fccc827da24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba8abfad-4308-4775-a81e-7fccc827da24\",\n \"name\": \"ba8abfad-4308-4775-a81e-7fccc827da24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:59:34.896590709Z\",\n \"endTime\": \"2026-10-02T22:18:38.199181809Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b86qkn-wdjflpkjvp8p-np-upgrade-b8-539690a9/providers/Microsoft.Compute/virtualMachines/j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b86qkn-wdjflpkjvp8p-np-upgrade-b8-539690a9/providers/Microsoft.Compute/virtualMachines/j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008d1b20\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-b86qkn\" in resourcegroup=\"rg-np-version-upgrade-b86qkn-wdjflpkjvp8p\" to finish creating: GET https://rp.j0034432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba8abfad-4308-4775-a81e-7fccc827da24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ba8abfad-4308-4775-a81e-7fccc827da24\",\n \"name\": \"ba8abfad-4308-4775-a81e-7fccc827da24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:59:34.896590709Z\",\n \"endTime\": \"2026-10-02T22:18:38.199181809Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b86qkn-wdjflpkjvp8p-np-upgrade-b8-539690a9/providers/Microsoft.Compute/virtualMachines/j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-b86qkn-wdjflpkjvp8p-np-upgrade-b8-539690a9/providers/Microsoft.Compute/virtualMachines/j2s1d7t0q5w9c1v-npupgrade-4-21-cfbt7-68vsc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"70d78761e859d173e39082a7e0f9097f17c46bd73f687febaabbe8dc7e726da0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106117061750034432","occurred_at":"2026-10-02T20:20:11Z","failed_at":"e2e","signature_id":"24d9756fc06393ff60a7f26c3d09a05911ada4575a0de58939f9e9ec06ce0777","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.006s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-d97ndt\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001ec6360\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.006s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-d97ndt\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001ec6360\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db42d3a8bb432b55cd45f2d44660dfafa02906a8443476f401d5f2c067022daa","failure_pattern":"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"7aa5b79d263d689bf60ede71bfb76af46cd24146cb8662a299a330b96cc2d23e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106117061750034432","occurred_at":"2026-10-02T20:20:11Z","failed_at":"e2e","signature_id":"5c9873debb2342170eed4903e7f228938c3a34d04b4f651f99d6b862372dfffb","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001047368\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001047368\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","db42d3a8bb432b55cd45f2d44660dfafa02906a8443476f401d5f2c067022daa"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7327/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106114529938116608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7327,"pr_state":"open","pr_sha":"087cc6716f0af6be78f9c78981dcaf50d19f4f8d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T20:10:08Z","started_at":"2026-10-02T20:10:08Z","completed_at":"2026-10-02T22:45:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T22:58:27.010656473Z","timing_metadata_checked_at":"2026-10-02T22:58:27.010656473Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T22:58:27.013324464Z","region_metadata_checked_at":"2026-10-02T22:58:27.013324464Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"00e6e7a09f9f45e9d50c113c8f702dde93bc9d01f9eccfdea3223a419dd0e82f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7327/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106114529938116608","occurred_at":"2026-10-02T20:10:08Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T21:34:19Z\n Ended: 2026-10-02T21:40:20Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8116608-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"153b1ae4fef893efbca85984488adc07926a02122e0fe98b379a847a9243ab9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7327/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106114529938116608","occurred_at":"2026-10-02T20:10:08Z","failed_at":"e2e","signature_id":"5b5e8add1f118eb1fc026e437f941636e24ad6dd0bfaf6584e1d203e456a64d8","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00022e5a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-6tx6ckwrt7wg\" to finish creating: GET https://rp.j8116608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b9f2f0a-a724-431a-8648-3c5581a50113\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b9f2f0a-a724-431a-8648-3c5581a50113\",\n \"name\": \"2b9f2f0a-a724-431a-8648-3c5581a50113\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:27:28.841238262Z\",\n \"endTime\": \"2026-10-02T21:46:29.997272093Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v3c9l1u3o0d3l2n-np-1-bhwvt-x7hn7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-6tx6ckwrt7wg--managed/providers/Microsoft.Compute/virtualMachines/v3c9l1u3o0d3l2n-np-1-bhwvt-x7hn7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v3c9l1u3o0d3l2n-np-1-bhwvt-x7hn7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-6tx6ckwrt7wg--managed/providers/Microsoft.Compute/virtualMachines/v3c9l1u3o0d3l2n-np-1-bhwvt-x7hn7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00022e5a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-6tx6ckwrt7wg\" to finish creating: GET https://rp.j8116608.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b9f2f0a-a724-431a-8648-3c5581a50113\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2b9f2f0a-a724-431a-8648-3c5581a50113\",\n \"name\": \"2b9f2f0a-a724-431a-8648-3c5581a50113\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:27:28.841238262Z\",\n \"endTime\": \"2026-10-02T21:46:29.997272093Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v3c9l1u3o0d3l2n-np-1-bhwvt-x7hn7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-6tx6ckwrt7wg--managed/providers/Microsoft.Compute/virtualMachines/v3c9l1u3o0d3l2n-np-1-bhwvt-x7hn7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v3c9l1u3o0d3l2n-np-1-bhwvt-x7hn7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-6tx6ckwrt7wg--managed/providers/Microsoft.Compute/virtualMachines/v3c9l1u3o0d3l2n-np-1-bhwvt-x7hn7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106113576442793984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7323,"pr_state":"closed","pr_sha":"e0cc9624c774a344398e0b62339c3bdfb03bc924","final_merged_sha":"e0cc9624c774a344398e0b62339c3bdfb03bc924","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T20:06:16Z","started_at":"2026-10-02T20:06:16Z","completed_at":"2026-10-02T22:43:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T22:58:26.800092153Z","timing_metadata_checked_at":"2026-10-02T22:58:26.800092153Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T22:58:26.803531442Z","region_metadata_checked_at":"2026-10-02T22:58:26.803531442Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106109176676618240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7326,"pr_state":"closed","pr_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","final_merged_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T19:48:51Z","started_at":"2026-10-02T19:48:51Z","completed_at":"2026-10-02T22:06:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T22:16:37.755531249Z","timing_metadata_checked_at":"2026-10-02T22:16:37.755531249Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T22:16:37.75827354Z","region_metadata_checked_at":"2026-10-02T22:16:37.75827354Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1dc695b8fd35ed1d346ce038e11db7d610e6e05fc53d1e0d22c2baa620a5a396","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106109176676618240","occurred_at":"2026-10-02T19:48:51Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T20:53:37Z\n Ended: 2026-10-02T20:59:37Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6618240-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"48f9b113e19049d0aaf7930cfa732148931491ab7d1a43916d076d6a9802d59e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106109176676618240","occurred_at":"2026-10-02T19:48:51Z","failed_at":"e2e","signature_id":"0b09878cead7ed0db7149a6d094f72740b9f49ac8eb39886c52d3d3e4afc2dc1","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00378d080\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-46hf4d98wdxd\" to finish creating: GET https://rp.j6618240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\",\n \"name\": \"19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:11:48.464360183Z\",\n \"endTime\": \"2026-10-02T21:30:51.922878069Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-46hf4d98wdxd--managed/providers/Microsoft.Compute/virtualMachines/s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-46hf4d98wdxd--managed/providers/Microsoft.Compute/virtualMachines/s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00378d080\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-46hf4d98wdxd\" to finish creating: GET https://rp.j6618240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\",\n \"name\": \"19cdee0b-c4b9-46e1-8ec8-1661a96d6bea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:11:48.464360183Z\",\n \"endTime\": \"2026-10-02T21:30:51.922878069Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-46hf4d98wdxd--managed/providers/Microsoft.Compute/virtualMachines/s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-46hf4d98wdxd--managed/providers/Microsoft.Compute/virtualMachines/s0m9z3c4v3c7d8p-np-delete-main-pqbrm-hxrzm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-46hf4d98wdxd\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106109131696902144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7309,"pr_state":"closed","pr_sha":"af8d2c9412c3aa9ef875de263742495a37299cc0","final_merged_sha":"af8d2c9412c3aa9ef875de263742495a37299cc0","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T19:48:41Z","started_at":"2026-10-02T19:48:41Z","completed_at":"2026-10-02T22:02:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T22:16:37.62127469Z","timing_metadata_checked_at":"2026-10-02T22:16:37.62127469Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T22:16:37.624329479Z","region_metadata_checked_at":"2026-10-02T22:16:37.624329479Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"77d3c736755d2fe00b84c2b64ef577320868c92108745973cbfa0ba70a56fd0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106109131696902144","occurred_at":"2026-10-02T19:48:41Z","failed_at":"e2e","signature_id":"f88616dce735dbe299ec31741c0e4e4dd95b26a9ca444ca47280bbcc48727d97","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-q47mgs\" for cluster \"cluster-candidate-4-21-q47mgs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00067a620\u003e: \n failed to create NodePool np-q47mgs: failed waiting for nodepool=\"np-q47mgs\" for cluster \"cluster-candidate-4-21-q47mgs\" in resourcegroup=\"rg-candidate-4-21-8nd2gr7hp9jn\" to finish creating: GET https://rp.j6902144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc132130-8033-44d4-826f-921357d1ad9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc132130-8033-44d4-826f-921357d1ad9d\",\n \"name\": \"fc132130-8033-44d4-826f-921357d1ad9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:44:13.243887943Z\",\n \"endTime\": \"2026-10-02T21:03:17.125285435Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q3s1n7f1d8j4z7u-np-q47mgs-sg5mc-vdkh2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8nd2gr7hp9jn-candidate-q47mgs--managed/providers/Microsoft.Compute/virtualMachines/q3s1n7f1d8j4z7u-np-q47mgs-sg5mc-vdkh2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q3s1n7f1d8j4z7u-np-q47mgs-sg5mc-vdkh2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8nd2gr7hp9jn-candidate-q47mgs--managed/providers/Microsoft.Compute/virtualMachines/q3s1n7f1d8j4z7u-np-q47mgs-sg5mc-vdkh2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-q47mgs\" for cluster \"cluster-candidate-4-21-q47mgs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00067a620\u003e: \n failed to create NodePool np-q47mgs: failed waiting for nodepool=\"np-q47mgs\" for cluster \"cluster-candidate-4-21-q47mgs\" in resourcegroup=\"rg-candidate-4-21-8nd2gr7hp9jn\" to finish creating: GET https://rp.j6902144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc132130-8033-44d4-826f-921357d1ad9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fc132130-8033-44d4-826f-921357d1ad9d\",\n \"name\": \"fc132130-8033-44d4-826f-921357d1ad9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:44:13.243887943Z\",\n \"endTime\": \"2026-10-02T21:03:17.125285435Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q3s1n7f1d8j4z7u-np-q47mgs-sg5mc-vdkh2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8nd2gr7hp9jn-candidate-q47mgs--managed/providers/Microsoft.Compute/virtualMachines/q3s1n7f1d8j4z7u-np-q47mgs-sg5mc-vdkh2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q3s1n7f1d8j4z7u-np-q47mgs-sg5mc-vdkh2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8nd2gr7hp9jn-candidate-q47mgs--managed/providers/Microsoft.Compute/virtualMachines/q3s1n7f1d8j4z7u-np-q47mgs-sg5mc-vdkh2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106096143061487616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7321,"pr_state":"closed","pr_sha":"c2eb4d5a17ddbf56ef3a8d378d7990dee66ec717","final_merged_sha":"c2eb4d5a17ddbf56ef3a8d378d7990dee66ec717","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T18:57:04Z","started_at":"2026-10-02T18:57:04Z","completed_at":"2026-10-02T21:13:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T21:24:29.317143128Z","timing_metadata_checked_at":"2026-10-02T21:24:29.317143128Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T21:24:29.321532513Z","region_metadata_checked_at":"2026-10-02T21:24:29.321532513Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"743fc292e76a539af1c737485d63212273f13e575d6193e24a179754c939a5ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106096143061487616","occurred_at":"2026-10-02T18:57:04Z","failed_at":"e2e","signature_id":"75c54a19ff7c88fe534fc97b238d4fa34e6bdf28dcd643fa62b5d5f8c5dde05b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001e820c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ftx74z7plhzf\" to finish creating: GET https://rp.j1487616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/737de5ed-e682-479d-bd22-1b9ebdbeab1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/737de5ed-e682-479d-bd22-1b9ebdbeab1a\",\n \"name\": \"737de5ed-e682-479d-bd22-1b9ebdbeab1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:53:28.478249577Z\",\n \"endTime\": \"2026-10-02T20:12:35.176603143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cmpgt-5gqm9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-5gqm9 to exist\\nMachine cilium-cluster-cilium-np-cmpgt-cg7p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-cg7p2 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cmpgt-5gqm9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-5gqm9 to exist\\nMachine cilium-cluster-cilium-np-cmpgt-cg7p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-cg7p2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001e820c0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-ftx74z7plhzf\" to finish creating: GET https://rp.j1487616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/737de5ed-e682-479d-bd22-1b9ebdbeab1a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/737de5ed-e682-479d-bd22-1b9ebdbeab1a\",\n \"name\": \"737de5ed-e682-479d-bd22-1b9ebdbeab1a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:53:28.478249577Z\",\n \"endTime\": \"2026-10-02T20:12:35.176603143Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-cmpgt-5gqm9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-5gqm9 to exist\\nMachine cilium-cluster-cilium-np-cmpgt-cg7p2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-cg7p2 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-cmpgt-5gqm9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-5gqm9 to exist\\nMachine cilium-cluster-cilium-np-cmpgt-cg7p2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-ftx74z7plhzf--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-cmpgt-cg7p2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106091840703827968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7311,"pr_state":"open","pr_sha":"825d85ae29257f2fab9ba0b75f3daf7790337077","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T18:39:58Z","started_at":"2026-10-02T18:39:58Z","completed_at":"2026-10-02T21:18:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T21:35:21.624097508Z","timing_metadata_checked_at":"2026-10-02T21:35:21.624097508Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T21:35:21.630604487Z","region_metadata_checked_at":"2026-10-02T21:35:21.630604487Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0d0fd842a4cfb03814d0eb53905bf2f5a71b3794625f35ab538b8f0c58ba980a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106091840703827968","occurred_at":"2026-10-02T18:39:58Z","failed_at":"alert","signature_id":"7deb638e8e55b55194c14aa0cd42ee58586132dbbb6f2d7f6d32b6dbaa200027","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 40 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T20:04:20Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-rqmvp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-rqmvp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-02T20:04:20Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-pclwm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-pclwm has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-ffmdk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-ffmdk has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-lqbwl\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-lqbwl has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:23:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-7xqdb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-7xqdb has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-xfn2j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-xfn2j has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-hrb67\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-hrb67 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-9rzfn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-9rzfn has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-8rlch\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-8rlch has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-xz6lm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-xz6lm has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-psr9c\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-psr9c has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-tdg92\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-tdg92 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-nnzlj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-nnzlj has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-cp4cg\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-cp4cg has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-zs2jk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-zs2jk has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-bft9l\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-bft9l has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-l6fg8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-l6fg8 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-nkfbd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-nkfbd has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-zqdt7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-zqdt7 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-6hf78\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-6hf78 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-v2424\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-v2424 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-7xlbs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-7xlbs has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-gr2nd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-gr2nd has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-hhkrx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-hhkrx has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:23:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-wnr5j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-wnr5j has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-6kxrp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-6kxrp has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-wxmwp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-wxmwp has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-dgvs2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-dgvs2 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-nt5f5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-nt5f5 has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-qldfs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-qldfs has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-vjmrq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-vjmrq has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:23:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-mz8lv\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-mz8lv has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-7r5rf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-7r5rf has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:23:24Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-4ljs8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-4ljs8 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-vp8px\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-vp8px has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-t8hh6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-t8hh6 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-9xv2x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-9xv2x has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-w4rxn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-w4rxn has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a\", pod=\"ignition-server-proxy-789f8b6d5-mfxhq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roevlt7isirqt9vh847qin5e1aq08-z1l9m0v5r8f6l2a/ignition-server-proxy-789f8b6d5-mfxhq has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-10-02T20:05:22Z\n Ended: 2026-10-02T20:18:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3827968-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b\", pod=\"ignition-server-proxy-5fc76ccd5b-vngdx\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6roerb25v628veke8v55ppk077aba6-u0o7a0t6f5u8t3b/ignition-server-proxy-5fc76ccd5b-vngdx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"6bd2a20e60fbbc30ff40b00b020c3385fbc845301eb7663c95a7a23d550e6b9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106091840703827968","occurred_at":"2026-10-02T18:39:58Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"9099049c0d47ac62641866e92f7dc3f0f9fd74f5e93819e844b1d421ffa87e93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7311/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106091840703827968","occurred_at":"2026-10-02T18:39:58Z","failed_at":"e2e","signature_id":"dfa343b19e8ca6f894f17713a054300c8d7a7c712659fa089dac380e5a5e1ab0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002838820\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc002838820\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106082635410313216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7305,"pr_state":"open","pr_sha":"0330fc4ed6fa561878e47fa8572ff178a1276c4b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T18:03:24Z","started_at":"2026-10-02T18:03:24Z","completed_at":"2026-10-02T20:27:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:41:54.327189979Z","timing_metadata_checked_at":"2026-10-02T20:41:54.327189979Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:41:54.339281945Z","region_metadata_checked_at":"2026-10-02T20:41:54.339281945Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"6087ebdbaeaa4d48a302905dc8bed01dc9280d32c0cc738401a0a427e7912abe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106082635410313216","occurred_at":"2026-10-02T18:03:24Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"f02c175568b6581e2d3356988f414faa6b96e99befedb150385f322de59bf27a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106082635410313216","occurred_at":"2026-10-02T18:03:24Z","failed_at":"e2e","signature_id":"a9519ed127543f85f80a4754e183aabc6e52e7d298e88306d50cff9591c5662c","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-c74lpp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001776360\u003e: \n DNS for route host agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-c74lpp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001776360\u003e: \n DNS for route host agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f7mjz.apps.aro.p7k6j4s9j1a9f2j.luhb.j0313216.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"fc30fd66386d95bb075c795cd8b7af20e842baf89ab8cbcaae31c9d181ed83f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7305/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106082635410313216","occurred_at":"2026-10-02T18:03:24Z","failed_at":"e2e","signature_id":"1983f452786e60abbdb725b8622697151a96361ebb27e00c6a2b0a244b8f9d74","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00099b300\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8jgnh62lstlc\" to finish creating: GET https://rp.j0313216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/baa8a5de-ef44-437c-8139-ba1877f5af42\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/baa8a5de-ef44-437c-8139-ba1877f5af42\",\n \"name\": \"baa8a5de-ef44-437c-8139-ba1877f5af42\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:34:49.936949596Z\",\n \"endTime\": \"2026-10-02T19:53:54.748504782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a8h7l0q0o6h7k0x-np-1-l7qq4-bqbqw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-8jgnh62lstlc--managed/providers/Microsoft.Compute/virtualMachines/a8h7l0q0o6h7k0x-np-1-l7qq4-bqbqw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a8h7l0q0o6h7k0x-np-1-l7qq4-bqbqw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-8jgnh62lstlc--managed/providers/Microsoft.Compute/virtualMachines/a8h7l0q0o6h7k0x-np-1-l7qq4-bqbqw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00099b300\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-8jgnh62lstlc\" to finish creating: GET https://rp.j0313216.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/baa8a5de-ef44-437c-8139-ba1877f5af42\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/baa8a5de-ef44-437c-8139-ba1877f5af42\",\n \"name\": \"baa8a5de-ef44-437c-8139-ba1877f5af42\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:34:49.936949596Z\",\n \"endTime\": \"2026-10-02T19:53:54.748504782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a8h7l0q0o6h7k0x-np-1-l7qq4-bqbqw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-8jgnh62lstlc--managed/providers/Microsoft.Compute/virtualMachines/a8h7l0q0o6h7k0x-np-1-l7qq4-bqbqw to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a8h7l0q0o6h7k0x-np-1-l7qq4-bqbqw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-8jgnh62lstlc--managed/providers/Microsoft.Compute/virtualMachines/a8h7l0q0o6h7k0x-np-1-l7qq4-bqbqw to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106081367363162112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6775,"pr_state":"closed","pr_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","final_merged_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T17:58:18Z","started_at":"2026-10-02T17:58:18Z","completed_at":"2026-10-02T20:33:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:41:53.982529047Z","timing_metadata_checked_at":"2026-10-02T20:41:53.982529047Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:41:53.986836635Z","region_metadata_checked_at":"2026-10-02T20:41:53.986836635Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"39982769b2f28123365709730f2481641d2222240b757d5db26aede13b63a400","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106081367363162112","occurred_at":"2026-10-02T17:58:18Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T19:11:45Z\n Ended: 2026-10-02T19:19:41Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3162112-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106081336937680896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"closed","pr_sha":"7868ef2b26e0bfedc24919742a75cbc3e71f2259","final_merged_sha":"bd1645798d81ff2bb96a575e2cfb1ce745182e60","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T17:58:14Z","started_at":"2026-10-02T17:58:14Z","completed_at":"2026-10-02T20:30:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:41:54.177638199Z","timing_metadata_checked_at":"2026-10-02T20:41:54.177638199Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:41:54.180700791Z","region_metadata_checked_at":"2026-10-02T20:41:54.180700791Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"205e7d0b510537f0b8ddc4408a98763d94fdd114cfd8d834488bf224a9b86045","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106081336937680896","occurred_at":"2026-10-02T17:58:14Z","failed_at":"e2e","signature_id":"31dc382fb43fe26f7ce7e41de756fcc58f0efe54d9f97db17cfd4bb76fb55a77","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c38480\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-54tj44\" in resourcegroup=\"rg-np-skip-minor-54tj44-rqt5qjd65czw\" to finish creating: GET https://rp.j7680896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d49abb2-b87c-4df6-b87c-df15e54da043\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d49abb2-b87c-4df6-b87c-df15e54da043\",\n \"name\": \"6d49abb2-b87c-4df6-b87c-df15e54da043\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:28:01.557139687Z\",\n \"endTime\": \"2026-10-02T19:47:08.946643419Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e5k1q1b6x4c4e8b-nps-420-422-g7bhw-kc47b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-54tj44-rqt5qjd65czw-np-sm-54tj44--managed/providers/Microsoft.Compute/virtualMachines/e5k1q1b6x4c4e8b-nps-420-422-g7bhw-kc47b to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5k1q1b6x4c4e8b-nps-420-422-g7bhw-kc47b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-54tj44-rqt5qjd65czw-np-sm-54tj44--managed/providers/Microsoft.Compute/virtualMachines/e5k1q1b6x4c4e8b-nps-420-422-g7bhw-kc47b to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c38480\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-54tj44\" in resourcegroup=\"rg-np-skip-minor-54tj44-rqt5qjd65czw\" to finish creating: GET https://rp.j7680896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d49abb2-b87c-4df6-b87c-df15e54da043\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6d49abb2-b87c-4df6-b87c-df15e54da043\",\n \"name\": \"6d49abb2-b87c-4df6-b87c-df15e54da043\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:28:01.557139687Z\",\n \"endTime\": \"2026-10-02T19:47:08.946643419Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e5k1q1b6x4c4e8b-nps-420-422-g7bhw-kc47b: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-54tj44-rqt5qjd65czw-np-sm-54tj44--managed/providers/Microsoft.Compute/virtualMachines/e5k1q1b6x4c4e8b-nps-420-422-g7bhw-kc47b to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5k1q1b6x4c4e8b-nps-420-422-g7bhw-kc47b: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-54tj44-rqt5qjd65czw-np-sm-54tj44--managed/providers/Microsoft.Compute/virtualMachines/e5k1q1b6x4c4e8b-nps-420-422-g7bhw-kc47b to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4b1f84e3451552b9038899146866b7da5f466576af52a11a7330546d52378ea1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106081336937680896","occurred_at":"2026-10-02T17:58:14Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T19:07:37Z\n Ended: 2026-10-02T19:15:36Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7680896-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106078475705126912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7323,"pr_state":"closed","pr_sha":"e0cc9624c774a344398e0b62339c3bdfb03bc924","final_merged_sha":"e0cc9624c774a344398e0b62339c3bdfb03bc924","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T17:46:52Z","started_at":"2026-10-02T17:46:52Z","completed_at":"2026-10-02T19:53:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:10:41.062622573Z","timing_metadata_checked_at":"2026-10-02T20:10:41.062622573Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:10:41.066086262Z","region_metadata_checked_at":"2026-10-02T20:10:41.066086262Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"8347ce30305d30aecf31951daafcff896af044fbf650a6ecb4c209eb0e29ebda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106078475705126912","occurred_at":"2026-10-02T17:46:52Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T18:37:05Z\n Ended: 2026-10-02T18:43:05Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5126912-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106077226553315328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"20352914bcc2fe997b89febb09bba5ef52a89abe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T17:41:54Z","started_at":"2026-10-02T17:41:54Z","completed_at":"2026-10-02T20:06:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:21:14.927903617Z","timing_metadata_checked_at":"2026-10-02T20:21:14.927903617Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:21:14.931322606Z","region_metadata_checked_at":"2026-10-02T20:21:14.931322606Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106065771653238784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7323,"pr_state":"closed","pr_sha":"e0cc9624c774a344398e0b62339c3bdfb03bc924","final_merged_sha":"e0cc9624c774a344398e0b62339c3bdfb03bc924","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T16:56:23Z","started_at":"2026-10-02T16:56:23Z","completed_at":"2026-10-02T17:27:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:41:29.027729392Z","timing_metadata_checked_at":"2026-10-02T17:41:29.027729392Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-10-02T17:41:29.032059179Z","region_metadata_checked_at":"2026-10-02T18:45:04.084077423Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"47424b85c55aed3595a6e8d3a96400a4b677df3f299c8ee576db8acbfb8ac6e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106065771653238784","occurred_at":"2026-10-02T16:56:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106064405874610176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7309,"pr_state":"closed","pr_sha":"af8d2c9412c3aa9ef875de263742495a37299cc0","final_merged_sha":"af8d2c9412c3aa9ef875de263742495a37299cc0","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T16:50:57Z","started_at":"2026-10-02T16:50:57Z","completed_at":"2026-10-02T19:29:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T19:38:35.316020616Z","timing_metadata_checked_at":"2026-10-02T19:38:35.316020616Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T19:38:35.319442305Z","region_metadata_checked_at":"2026-10-02T19:38:35.319442305Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"422db98c52ca5acd1b658fe2d8437167bd29806d06bcaabb7801254bb9d742e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106064405874610176","occurred_at":"2026-10-02T16:50:57Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T17:55:55Z\n Ended: 2026-10-02T18:01:55Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4610176-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"c485d97cff7da7c5d19b69f27d5256fae0ac49c58066e0e4d3a123f5cda65971","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106064405874610176","occurred_at":"2026-10-02T16:50:57Z","failed_at":"alert","signature_id":"c2fcc826cd885e26911af5d73a5995409a57254fd3df2439b6894c3199cddb54","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T19:11:26Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4610176-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.77:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4610176/providers/microsoft.monitor/accounts/services-j4610176\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-69f9b555fd-7q6cb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4610176-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-02T19:11:26Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4610176-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.77:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4610176/providers/microsoft.monitor/accounts/services-j4610176\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-69f9b555fd-7q6cb\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4610176-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106063651092828160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7326,"pr_state":"closed","pr_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","final_merged_sha":"a1ca37f98e00a3ceda2e6f4424374b44a32589fb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T16:47:51Z","started_at":"2026-10-02T16:47:51Z","completed_at":"2026-10-02T19:28:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T19:38:35.466044547Z","timing_metadata_checked_at":"2026-10-02T19:38:35.466044547Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T19:38:35.468830738Z","region_metadata_checked_at":"2026-10-02T19:38:35.468830738Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"10fde55d0dd63cdd8d792b85c6ced0e7df8eea68397a2a90bf8c1f2b1569a0da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106063651092828160","occurred_at":"2026-10-02T16:47:51Z","failed_at":"alert","signature_id":"8a0e5803002408b9740f03b409808525c6d5017ddf44ce6a4187ef72194ff101","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 65 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-22dl9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-22dl9 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-dfssq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-dfssq has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-lc7f8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-lc7f8 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-lrxrn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-lrxrn has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-ws459\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-ws459 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-nm4b8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-nm4b8 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-r88sm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-r88sm has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-2f87w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-2f87w has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-hlcf4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-hlcf4 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-6mnrg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-6mnrg has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-t95zp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-t95zp has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-jkbjb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-jkbjb has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-zcpgg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-zcpgg has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-qxvct\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-qxvct has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-mfbkm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-mfbkm has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-mmpwv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-mmpwv has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-m4v82\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-m4v82 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-fzf98\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-fzf98 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-6f2b7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-6f2b7 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-z9cp6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-z9cp6 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-64mk2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-64mk2 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-glxpz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-glxpz has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-scxrj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-scxrj has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-5876w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-5876w has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-h5skh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-h5skh has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-77rvv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-77rvv has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-wxzgc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-wxzgc has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-gxvrp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-gxvrp has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-f497d\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-f497d has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-zpfft\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-zpfft has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-x99lx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-x99lx has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-ksr8r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-ksr8r has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-p2k9c\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-p2k9c has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-hp5fb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-hp5fb has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-vjh72\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-vjh72 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-gfrgk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-gfrgk has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-nrh6c\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-nrh6c has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-dcz7s\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-dcz7s has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-56n2t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-56n2t has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-8pwww\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-8pwww has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-sr95r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-sr95r has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-sl2gh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-sl2gh has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-dlbhp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-dlbhp has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-pbjdw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-pbjdw has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-rfjgp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-rfjgp has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-xwfwj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-xwfwj has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-gl2xg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-gl2xg has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-96lg5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-96lg5 has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-vbcxj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-vbcxj has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-5sw59\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-5sw59 has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-l49jm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-l49jm has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-jx5v4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-jx5v4 has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-s256n\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-s256n has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-vnr48\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-vnr48 has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-zrk5n\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-zrk5n has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-cc7rs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-cc7rs has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-6j84r\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-6j84r has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-blmqs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-blmqs has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-4lnm2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-4lnm2 has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-ppjmp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-ppjmp has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-t9nwg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-t9nwg has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-68bhc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-68bhc has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-5fhnt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-5fhnt has been in a non-ready state for longer than 5 minutes.\nFiring 64:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-jjrn5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-jjrn5 has been in a non-ready state for longer than 5 minutes.\nFiring 65:\n State: Fired (not resolved)\n Started: 2026-10-02T17:38:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2828160-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-rzbk6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-rzbk6 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"85db8bfe60ce8db59f7d32b16a878a18344ebf3f99f23b0c203e67ce6a7585c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106063651092828160","occurred_at":"2026-10-02T16:47:51Z","failed_at":"e2e","signature_id":"80687d33a4d52dba803f3167e5984e21a97d039dd3cf4af22716d553b7595b37","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7b9c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-92jbnctz6g9n\" to finish creating: GET https://rp.j2828160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16cd1ed0-a68c-40a4-ac3b-14d23a71ccf8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16cd1ed0-a68c-40a4-ac3b-14d23a71ccf8\",\n \"name\": \"16cd1ed0-a68c-40a4-ac3b-14d23a71ccf8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T17:47:10.869193334Z\",\n \"endTime\": \"2026-10-02T18:06:19.043011596Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e0f5y6t9v5u2s0y-np-1-5bmkp-xqtnd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-92jbnctz6g9n--managed/providers/Microsoft.Compute/virtualMachines/e0f5y6t9v5u2s0y-np-1-5bmkp-xqtnd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e0f5y6t9v5u2s0y-np-1-5bmkp-xqtnd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-92jbnctz6g9n--managed/providers/Microsoft.Compute/virtualMachines/e0f5y6t9v5u2s0y-np-1-5bmkp-xqtnd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:178]: failed to create node pool np-1 for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7b9c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-92jbnctz6g9n\" to finish creating: GET https://rp.j2828160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16cd1ed0-a68c-40a4-ac3b-14d23a71ccf8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/16cd1ed0-a68c-40a4-ac3b-14d23a71ccf8\",\n \"name\": \"16cd1ed0-a68c-40a4-ac3b-14d23a71ccf8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T17:47:10.869193334Z\",\n \"endTime\": \"2026-10-02T18:06:19.043011596Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e0f5y6t9v5u2s0y-np-1-5bmkp-xqtnd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-92jbnctz6g9n--managed/providers/Microsoft.Compute/virtualMachines/e0f5y6t9v5u2s0y-np-1-5bmkp-xqtnd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e0f5y6t9v5u2s0y-np-1-5bmkp-xqtnd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-92jbnctz6g9n--managed/providers/Microsoft.Compute/virtualMachines/e0f5y6t9v5u2s0y-np-1-5bmkp-xqtnd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c93f256ecb9a3973d5f22c8608ab36080d82f715ecdd328a73020d4c86d61517","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7326/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106063651092828160","occurred_at":"2026-10-02T16:47:51Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T17:55:13Z\n Ended: 2026-10-02T18:02:17Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2828160-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106055090056990720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"closed","pr_sha":"7da218ee42bad349e24419fd993865ad4b1fe8ad","final_merged_sha":"ea2df83db19482a983a29965803a71b1f8158ab7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T16:13:56Z","started_at":"2026-10-02T16:13:56Z","completed_at":"2026-10-02T18:50:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T19:06:20.114629506Z","timing_metadata_checked_at":"2026-10-02T19:06:20.114629506Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T19:06:20.122866781Z","region_metadata_checked_at":"2026-10-02T19:06:20.122866781Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"107a647a589ca9e0fc15131fbc06503abcfae77f12ab972a1adbf0d09657bda8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106055090056990720","occurred_at":"2026-10-02T16:13:56Z","failed_at":"alert","signature_id":"9af0e1f2b1b242d263756ff45f69a3d0f7754608d5789a9bf850d6d14b180404","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T17:21:19Z\n Ended: 2026-10-02T17:32:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6990720-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6pa7oq39hmo04tlm81048rosc9slps-ea-listt98lhd\", pod=\"kube-apiserver-594c7bf48-44xbg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6pa7oq39hmo04tlm81048rosc9slps-ea-listt98lhd/kube-apiserver-594c7bf48-44xbg has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"60cd1d4e4d3d92b223a108dc5fb714dfe8eb82fe6f6a421b76e0fc4b45c39f9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106055090056990720","occurred_at":"2026-10-02T16:13:56Z","failed_at":"e2e","signature_id":"fd4d0de36426d6f1b5142554b466d73d39b32478872d976808a351b06917f2e4","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011448e8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011448e8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"e03b515ae29fd2703b3ee27cc399bf69765fc534078b511ac128acd151b92e92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106055090056990720","occurred_at":"2026-10-02T16:13:56Z","failed_at":"e2e","signature_id":"0a5b2526c26b68cdf55e2a3874db601a881bf79208555f69ca0b1f64f834031b","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.001s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-qjs7pm\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011ce348\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.001s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-qjs7pm\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0011ce348\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db42d3a8bb432b55cd45f2d44660dfafa02906a8443476f401d5f2c067022daa","failure_pattern":"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","db42d3a8bb432b55cd45f2d44660dfafa02906a8443476f401d5f2c067022daa"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106050489337516032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7321,"pr_state":"closed","pr_sha":"c2eb4d5a17ddbf56ef3a8d378d7990dee66ec717","final_merged_sha":"c2eb4d5a17ddbf56ef3a8d378d7990dee66ec717","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T15:55:39Z","started_at":"2026-10-02T15:55:39Z","completed_at":"2026-10-02T18:20:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T18:34:23.8401385Z","timing_metadata_checked_at":"2026-10-02T18:34:23.8401385Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T18:34:23.84325519Z","region_metadata_checked_at":"2026-10-02T18:34:23.84325519Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"cb59debc8e3895b50ed353408a5923787236f8b4ec85bb00bf5ef16d2c975053","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106050489337516032","occurred_at":"2026-10-02T15:55:39Z","failed_at":"alert","signature_id":"cbd61d65e78f3f789e94908c66dbeea403e2c88400610aac07b5bbbaef27053e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T17:02:52Z\n Ended: 2026-10-02T17:08:49Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7516032-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7296/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106047974684495872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7296,"pr_state":"open","pr_sha":"9cbe4a19beafc2c7218b5eceb0d902e1bd547d90","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T15:45:40Z","started_at":"2026-10-02T15:45:40Z","completed_at":"2026-10-02T18:26:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T18:34:23.544039118Z","timing_metadata_checked_at":"2026-10-02T18:34:23.544039118Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T18:34:23.548219905Z","region_metadata_checked_at":"2026-10-02T18:34:23.548219905Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7304/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106044631815819264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7304,"pr_state":"closed","pr_sha":"97468ac526e904de834c116bd25cfcebc4e5a1eb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T15:32:23Z","started_at":"2026-10-02T15:32:23Z","completed_at":"2026-10-02T17:57:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T18:13:18.784341398Z","timing_metadata_checked_at":"2026-10-02T18:13:18.784341398Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T18:13:18.788428386Z","region_metadata_checked_at":"2026-10-02T18:13:18.788428386Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7317/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106044264692584448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7317,"pr_state":"closed","pr_sha":"7717dd00dcdec243eaed5fb7962fc45f252e6b06","final_merged_sha":"7717dd00dcdec243eaed5fb7962fc45f252e6b06","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T15:30:50Z","started_at":"2026-10-02T15:30:50Z","completed_at":"2026-10-02T17:50:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T18:02:42.081270544Z","timing_metadata_checked_at":"2026-10-02T18:02:42.081270544Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T18:02:42.090306316Z","region_metadata_checked_at":"2026-10-02T18:02:42.090306316Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106039082894233600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"20352914bcc2fe997b89febb09bba5ef52a89abe","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T15:10:20Z","started_at":"2026-10-02T15:10:20Z","completed_at":"2026-10-02T17:35:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:51:43.788665905Z","timing_metadata_checked_at":"2026-10-02T17:51:43.788665905Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T17:51:43.791570496Z","region_metadata_checked_at":"2026-10-02T17:51:43.791570496Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0b649b7413097a60b4e9e3592076b06ed9f5c1262ae114d53caec48d115252d3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106039082894233600","occurred_at":"2026-10-02T15:10:20Z","failed_at":"e2e","signature_id":"6bce40273864c881050d6e2d7a4a0595d348985d0aa9ebd379d5a02487091686","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca0a60\u003e: \n failed to create HCP cluster ea-listpkp2p7: failed waiting for cluster=\"ea-listpkp2p7\" in resourcegroup=\"ea-list-4ccm8pbgrf47\" to finish creating: GET https://rp.j4233600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\",\n \"name\": \"ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:58:05.107845034Z\",\n \"endTime\": \"2026-10-02T16:27:08.62997793Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 1m14s ago; version 4.21.36 is Partial (want Completed), started 5m36s ago; version 4.21.36 is Partial (want Completed), started 11m4s ago; version 4.21.36 is Partial (want Completed), started 15m6s ago; version 4.21.36 is Partial (want Completed), started 20m46s ago; version 4.21.36 is Partial (want Completed), started 24m30s ago; version 4.21.36 is Partial (want Completed), started 25m39s ago; version 4.21.36 is Partial (want Completed), started 25m51s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca0a60\u003e: \n failed to create HCP cluster ea-listpkp2p7: failed waiting for cluster=\"ea-listpkp2p7\" in resourcegroup=\"ea-list-4ccm8pbgrf47\" to finish creating: GET https://rp.j4233600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\",\n \"name\": \"ff4033f3-93ba-43b3-86a5-c0cc8837ee8a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:58:05.107845034Z\",\n \"endTime\": \"2026-10-02T16:27:08.62997793Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.36 is Partial (want Completed), started 1m14s ago; version 4.21.36 is Partial (want Completed), started 5m36s ago; version 4.21.36 is Partial (want Completed), started 11m4s ago; version 4.21.36 is Partial (want Completed), started 15m6s ago; version 4.21.36 is Partial (want Completed), started 20m46s ago; version 4.21.36 is Partial (want Completed), started 24m30s ago; version 4.21.36 is Partial (want Completed), started 25m39s ago; version 4.21.36 is Partial (want Completed), started 25m51s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9875340340e8a84eab3c14c72cc498c82582aaf6c0fdf1646fe5c4ca338b7548","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7828c5be4625d636b8518361195a0eb8aa0c68d86a132936774a346421d4bb86","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106039082894233600","occurred_at":"2026-10-02T15:10:20Z","failed_at":"e2e","signature_id":"6c5905a125f03a9ccb5e9ff9fb30a16eb39c975e893c7c332e674d06be66eefe","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014427a0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h248xrjjcrlg\" to finish creating: GET https://rp.j4233600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38b2c7b6-a5ca-4ed6-a22c-bfc22444845b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38b2c7b6-a5ca-4ed6-a22c-bfc22444845b\",\n \"name\": \"38b2c7b6-a5ca-4ed6-a22c-bfc22444845b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T16:37:00.371020776Z\",\n \"endTime\": \"2026-10-02T16:56:08.531132296Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine d6z7a4r5k4r5t0t-autoscale-noaz-8nr45-nxs6x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-h248xrjjcrlg--managed/providers/Microsoft.Compute/virtualMachines/d6z7a4r5k4r5t0t-autoscale-noaz-8nr45-nxs6x to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine d6z7a4r5k4r5t0t-autoscale-noaz-8nr45-nxs6x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-h248xrjjcrlg--managed/providers/Microsoft.Compute/virtualMachines/d6z7a4r5k4r5t0t-autoscale-noaz-8nr45-nxs6x to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014427a0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-h248xrjjcrlg\" to finish creating: GET https://rp.j4233600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38b2c7b6-a5ca-4ed6-a22c-bfc22444845b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38b2c7b6-a5ca-4ed6-a22c-bfc22444845b\",\n \"name\": \"38b2c7b6-a5ca-4ed6-a22c-bfc22444845b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T16:37:00.371020776Z\",\n \"endTime\": \"2026-10-02T16:56:08.531132296Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine d6z7a4r5k4r5t0t-autoscale-noaz-8nr45-nxs6x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-h248xrjjcrlg--managed/providers/Microsoft.Compute/virtualMachines/d6z7a4r5k4r5t0t-autoscale-noaz-8nr45-nxs6x to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine d6z7a4r5k4r5t0t-autoscale-noaz-8nr45-nxs6x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-h248xrjjcrlg--managed/providers/Microsoft.Compute/virtualMachines/d6z7a4r5k4r5t0t-autoscale-noaz-8nr45-nxs6x to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8377ff4026c4602fc1b26b8c19791bd4090b14626e643bf9907ae77d6c0b25ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106039082894233600","occurred_at":"2026-10-02T15:10:20Z","failed_at":"e2e","signature_id":"03ee416419f4f30380384e985dc45aaa36ef7f2e21c2accfb48e05ac0da1c153","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bdc160\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bdc160\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"fbaa93ff61dd27b92ed5c2e1b74609b794a1852e678cff621b126caac9298a49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106039082894233600","occurred_at":"2026-10-02T15:10:20Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T16:13:37Z\n Ended: 2026-10-02T16:20:36Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4233600-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","9875340340e8a84eab3c14c72cc498c82582aaf6c0fdf1646fe5c4ca338b7548"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035454687055872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"closed","pr_sha":"7868ef2b26e0bfedc24919742a75cbc3e71f2259","final_merged_sha":"bd1645798d81ff2bb96a575e2cfb1ce745182e60","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T14:55:55Z","started_at":"2026-10-02T14:55:55Z","completed_at":"2026-10-02T17:36:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:51:43.552847974Z","timing_metadata_checked_at":"2026-10-02T17:51:43.552847974Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T17:51:43.556402263Z","region_metadata_checked_at":"2026-10-02T17:51:43.556402263Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ab1a1fc6a557fe0fe6c072902e6a97e86a62ed78bafaaf61289adef24c6d21b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035454687055872","occurred_at":"2026-10-02T14:55:55Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T16:10:23Z\n Ended: 2026-10-02T16:17:22Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7055872-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035428787228672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7314,"pr_state":"closed","pr_sha":"bbd92b0389c50f2199ac2153f3a2eadd01fee8a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T14:55:49Z","started_at":"2026-10-02T14:55:49Z","completed_at":"2026-10-02T17:34:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:51:43.981411676Z","timing_metadata_checked_at":"2026-10-02T17:51:43.981411676Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T17:51:43.984433266Z","region_metadata_checked_at":"2026-10-02T17:51:43.984433266Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2d81eca6b9b32d2e97710026cdbbaf196c18e53887e085b1497f20db60f485be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035428787228672","occurred_at":"2026-10-02T14:55:49Z","failed_at":"e2e","signature_id":"bc5be3ac53788be83982476868c7fc5ebab96a3431e2d77151fdef86774b0791","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011632a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-sq4gl58phslg\" to finish creating: GET https://rp.j7228672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b3769b3-98f7-4259-8915-bd5c46b08c2e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b3769b3-98f7-4259-8915-bd5c46b08c2e\",\n \"name\": \"1b3769b3-98f7-4259-8915-bd5c46b08c2e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T16:32:42.711328508Z\",\n \"endTime\": \"2026-10-02T16:51:42.25861049Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w8z1t1s1w4m9z0y-np-1-m2f5m-dvh7s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-sq4gl58phslg--managed/providers/Microsoft.Compute/virtualMachines/w8z1t1s1w4m9z0y-np-1-m2f5m-dvh7s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w8z1t1s1w4m9z0y-np-1-m2f5m-dvh7s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-sq4gl58phslg--managed/providers/Microsoft.Compute/virtualMachines/w8z1t1s1w4m9z0y-np-1-m2f5m-dvh7s to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011632a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-sq4gl58phslg\" to finish creating: GET https://rp.j7228672.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b3769b3-98f7-4259-8915-bd5c46b08c2e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1b3769b3-98f7-4259-8915-bd5c46b08c2e\",\n \"name\": \"1b3769b3-98f7-4259-8915-bd5c46b08c2e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T16:32:42.711328508Z\",\n \"endTime\": \"2026-10-02T16:51:42.25861049Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w8z1t1s1w4m9z0y-np-1-m2f5m-dvh7s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-sq4gl58phslg--managed/providers/Microsoft.Compute/virtualMachines/w8z1t1s1w4m9z0y-np-1-m2f5m-dvh7s to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w8z1t1s1w4m9z0y-np-1-m2f5m-dvh7s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-sq4gl58phslg--managed/providers/Microsoft.Compute/virtualMachines/w8z1t1s1w4m9z0y-np-1-m2f5m-dvh7s to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035412379111424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6775,"pr_state":"closed","pr_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","final_merged_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T14:55:45Z","started_at":"2026-10-02T14:55:45Z","completed_at":"2026-10-02T17:36:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:51:43.342486761Z","timing_metadata_checked_at":"2026-10-02T17:51:43.342486761Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T17:51:43.345996849Z","region_metadata_checked_at":"2026-10-02T17:51:43.345996849Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"04bcafe22c384f89b7fed684b88e6b53c492b7d5dd2a40158579ee23609d69fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035412379111424","occurred_at":"2026-10-02T14:55:45Z","failed_at":"alert","signature_id":"c4307b9de315687a8fce104eff903477686b7321152ba52d83a069c63664370a","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T15:56:30Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9111424-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-k7gkp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-k7gkp has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"6ec2ca3849d4d9a9ba669de0ecd4f55e4e5d81a2accc66485969cf6f4dee361f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035412379111424","occurred_at":"2026-10-02T14:55:45Z","failed_at":"alert","signature_id":"6c5480d9d7704b8a0e81ba15f14a4d9edb6d6b1ff0aff158900e2a7bd3221612","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 14 time(s) (2 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T16:07:06Z\n Ended: 2026-10-02T16:28:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9111424-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6ocek7h2b81l310othj19g5cs2hvdm-h6f8s9c6d5a8t7c\", pod=\"router-7466989cdd-4xgts\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6ocek7h2b81l310othj19g5cs2hvdm-h6f8s9c6d5a8t7c/router-7466989cdd-4xgts has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-02T16:07:07Z\n Ended: 2026-10-02T16:28:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9111424-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6oclvhiuo0ben6245b5rc60j4ul3v1-v20261001\", pod=\"router-58b59f844f-6fgw8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6oclvhiuo0ben6245b5rc60j4ul3v1-v20261001/router-58b59f844f-6fgw8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"952e20be415bc92888a7387a6e45e717e0d37fd02aa511b70cc891805d5bc423","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035412379111424","occurred_at":"2026-10-02T14:55:45Z","failed_at":"e2e","signature_id":"e55a99665469ae5bdd51816697d8d92b366d93c347a1df665aa039f3c7dfc45f","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-4-23-tjxdml\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8a7e0\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-4-23-tjxdml\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-23-klzk5dvhl6cx\" to finish creating: GET https://rp.j9111424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ab050f-9ade-44b7-9470-1944f6bc1377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ab050f-9ade-44b7-9470-1944f6bc1377\",\n \"name\": \"37ab050f-9ade-44b7-9470-1944f6bc1377\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T16:08:46.628951449Z\",\n \"endTime\": \"2026-10-02T16:27:53.982628223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z0i9p2v8n7r1u8z-np-deny-pods-hq6c9-vlq6k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-4-23-klzk5dvhl6cx-cp-ystream-de-fb92fcf5/providers/Microsoft.Compute/virtualMachines/z0i9p2v8n7r1u8z-np-deny-pods-hq6c9-vlq6k to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z0i9p2v8n7r1u8z-np-deny-pods-hq6c9-vlq6k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-4-23-klzk5dvhl6cx-cp-ystream-de-fb92fcf5/providers/Microsoft.Compute/virtualMachines/z0i9p2v8n7r1u8z-np-deny-pods-hq6c9-vlq6k to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:201]: failed to create node pool \"np-deny-pods\" for cluster \"cp-ystream-deny-pods-4-23-tjxdml\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8a7e0\u003e: \n failed to create NodePool np-deny-pods: failed waiting for nodepool=\"np-deny-pods\" for cluster \"cp-ystream-deny-pods-4-23-tjxdml\" in resourcegroup=\"rg-cp-ystream-deny-pods-4-23-klzk5dvhl6cx\" to finish creating: GET https://rp.j9111424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ab050f-9ade-44b7-9470-1944f6bc1377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37ab050f-9ade-44b7-9470-1944f6bc1377\",\n \"name\": \"37ab050f-9ade-44b7-9470-1944f6bc1377\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T16:08:46.628951449Z\",\n \"endTime\": \"2026-10-02T16:27:53.982628223Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z0i9p2v8n7r1u8z-np-deny-pods-hq6c9-vlq6k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-4-23-klzk5dvhl6cx-cp-ystream-de-fb92fcf5/providers/Microsoft.Compute/virtualMachines/z0i9p2v8n7r1u8z-np-deny-pods-hq6c9-vlq6k to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z0i9p2v8n7r1u8z-np-deny-pods-hq6c9-vlq6k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-deny-pods-4-23-klzk5dvhl6cx-cp-ystream-de-fb92fcf5/providers/Microsoft.Compute/virtualMachines/z0i9p2v8n7r1u8z-np-deny-pods-hq6c9-vlq6k to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7324/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106035227498385408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7324,"pr_state":"closed","pr_sha":"94272e840025a31c378e9d9be15fb47b8ae56fa8","final_merged_sha":"94272e840025a31c378e9d9be15fb47b8ae56fa8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T14:55:01Z","started_at":"2026-10-02T14:55:01Z","completed_at":"2026-10-02T17:22:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:30:48.220919777Z","timing_metadata_checked_at":"2026-10-02T17:30:48.220919777Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T17:30:48.224516066Z","region_metadata_checked_at":"2026-10-02T17:30:48.224516066Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106027308732125184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7323,"pr_state":"closed","pr_sha":"e0cc9624c774a344398e0b62339c3bdfb03bc924","final_merged_sha":"e0cc9624c774a344398e0b62339c3bdfb03bc924","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T14:23:33Z","started_at":"2026-10-02T14:23:33Z","completed_at":"2026-10-02T16:38:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T16:49:07.203560924Z","timing_metadata_checked_at":"2026-10-02T16:49:07.203560924Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T16:49:07.209535906Z","region_metadata_checked_at":"2026-10-02T16:49:07.209535906Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6df0a7f3d2ffe1dadc8e9195d063f5a51e77f5ea55abef4791b004153d498ace","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106027308732125184","occurred_at":"2026-10-02T14:23:33Z","failed_at":"e2e","signature_id":"a03b4f13d1b794c0ce5f7e27910be2a2b3cf8e2efc010caf5845fbed77edcc07","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xbxlzs\" for cluster \"cluster-candidate-4-20-xbxlzs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb3a80\u003e: \n failed to create NodePool np-xbxlzs: failed waiting for nodepool=\"np-xbxlzs\" for cluster \"cluster-candidate-4-20-xbxlzs\" in resourcegroup=\"rg-candidate-4-20-gwcqh9m2pngx\" to finish creating: GET https://rp.j2125184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e698a08-7d53-4eac-b66e-e0b997094971\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e698a08-7d53-4eac-b66e-e0b997094971\",\n \"name\": \"2e698a08-7d53-4eac-b66e-e0b997094971\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:16:45.725296348Z\",\n \"endTime\": \"2026-10-02T15:35:49.304545041Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w9p4z8q1b2g0x7t-np-xbxlzs-7kn5c-mb724: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwcqh9m2pngx-candidate-xbxlzs--managed/providers/Microsoft.Compute/virtualMachines/w9p4z8q1b2g0x7t-np-xbxlzs-7kn5c-mb724 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w9p4z8q1b2g0x7t-np-xbxlzs-7kn5c-mb724: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwcqh9m2pngx-candidate-xbxlzs--managed/providers/Microsoft.Compute/virtualMachines/w9p4z8q1b2g0x7t-np-xbxlzs-7kn5c-mb724 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xbxlzs\" for cluster \"cluster-candidate-4-20-xbxlzs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb3a80\u003e: \n failed to create NodePool np-xbxlzs: failed waiting for nodepool=\"np-xbxlzs\" for cluster \"cluster-candidate-4-20-xbxlzs\" in resourcegroup=\"rg-candidate-4-20-gwcqh9m2pngx\" to finish creating: GET https://rp.j2125184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e698a08-7d53-4eac-b66e-e0b997094971\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2e698a08-7d53-4eac-b66e-e0b997094971\",\n \"name\": \"2e698a08-7d53-4eac-b66e-e0b997094971\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:16:45.725296348Z\",\n \"endTime\": \"2026-10-02T15:35:49.304545041Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w9p4z8q1b2g0x7t-np-xbxlzs-7kn5c-mb724: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwcqh9m2pngx-candidate-xbxlzs--managed/providers/Microsoft.Compute/virtualMachines/w9p4z8q1b2g0x7t-np-xbxlzs-7kn5c-mb724 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w9p4z8q1b2g0x7t-np-xbxlzs-7kn5c-mb724: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-gwcqh9m2pngx-candidate-xbxlzs--managed/providers/Microsoft.Compute/virtualMachines/w9p4z8q1b2g0x7t-np-xbxlzs-7kn5c-mb724 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b63e2fdd7a11c377bfee53de264d55c2c02c8d56e99c24cc94b308402ad7166e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7323/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106027308732125184","occurred_at":"2026-10-02T14:23:33Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T15:21:16Z\n Ended: 2026-10-02T15:28:15Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2125184-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106024077666816000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"7208e20741be725f813b586b79cf64421524f49e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T14:10:42Z","started_at":"2026-10-02T14:10:42Z","completed_at":"2026-10-02T16:22:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T16:38:43.561277256Z","timing_metadata_checked_at":"2026-10-02T16:38:43.561277256Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T16:38:43.564624745Z","region_metadata_checked_at":"2026-10-02T16:38:43.564624745Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b1ddb8654e4cf4e37af2d31789d099d169ce8a0d58cb4b17d18697858e8df04d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106024077666816000","occurred_at":"2026-10-02T14:10:42Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T15:09:09Z\n Ended: 2026-10-02T15:15:04Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j6816000-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106020738229080064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7309,"pr_state":"closed","pr_sha":"af8d2c9412c3aa9ef875de263742495a37299cc0","final_merged_sha":"af8d2c9412c3aa9ef875de263742495a37299cc0","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T13:57:26Z","started_at":"2026-10-02T13:57:26Z","completed_at":"2026-10-02T16:23:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T16:38:43.871986663Z","timing_metadata_checked_at":"2026-10-02T16:38:43.871986663Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T16:38:43.874677254Z","region_metadata_checked_at":"2026-10-02T16:38:43.874677254Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"7ae995b55f0ce155f54d1b7dd36b7cdb7b3ac9dcaa516a74d07b9f16b49af42e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106020738229080064","occurred_at":"2026-10-02T13:57:26Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T15:01:55Z\n Ended: 2026-10-02T15:07:57Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9080064-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106003628014702592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7292,"pr_state":"open","pr_sha":"40ca0b770e0f74533c8da09a5069d4ba7295b76d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T12:49:27Z","started_at":"2026-10-02T12:49:27Z","completed_at":"2026-10-02T15:11:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T15:26:25.771649625Z","timing_metadata_checked_at":"2026-10-02T15:26:25.771649625Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T15:26:25.77505731Z","region_metadata_checked_at":"2026-10-02T15:26:25.77505731Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a64f94bd1e5c7f7ffd51728744677ec209216d1a55b767f8fcabdbb4cc9fe3dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106003628014702592","occurred_at":"2026-10-02T12:49:27Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T13:57:08Z\n Ended: 2026-10-02T14:15:08Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4702592-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106002969722884096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7321,"pr_state":"closed","pr_sha":"c2eb4d5a17ddbf56ef3a8d378d7990dee66ec717","final_merged_sha":"c2eb4d5a17ddbf56ef3a8d378d7990dee66ec717","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T12:46:50Z","started_at":"2026-10-02T12:46:50Z","completed_at":"2026-10-02T15:13:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T15:26:25.934419717Z","timing_metadata_checked_at":"2026-10-02T15:26:25.934419717Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T15:26:25.937311205Z","region_metadata_checked_at":"2026-10-02T15:26:25.937311205Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"739d991692330543c554670c87f3eaa6c670e680115bc75c9d91b49c10123e89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7321/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2106002969722884096","occurred_at":"2026-10-02T12:46:50Z","failed_at":"e2e","signature_id":"0d41539b045a3d8d46974b35f8490c28db0d1f05714109888ade97dba1eae56d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00156e168\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:197]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00156e168\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sxczr.apps.aro.cilium-cluster.5eji.j2884096.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105989665596116992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6775,"pr_state":"closed","pr_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","final_merged_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T11:53:58Z","started_at":"2026-10-02T11:53:58Z","completed_at":"2026-10-02T14:21:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T14:33:32.73764262Z","timing_metadata_checked_at":"2026-10-02T14:33:32.73764262Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T14:33:32.741593012Z","region_metadata_checked_at":"2026-10-02T14:33:32.741593012Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b95d647dbe5638c61591169b003f153865004c17496d57a9f09470cbc5df86f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105989665596116992","occurred_at":"2026-10-02T11:53:58Z","failed_at":"e2e","signature_id":"aca469fe140764c02ab7856c4660d7eff0c7a2566b4afda4da099d345dece618","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a14dc0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4jgj2xjf9rbf\" to finish creating: GET https://rp.j6116992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c833939d-30b9-4409-82fd-33d2b22e8826\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c833939d-30b9-4409-82fd-33d2b22e8826\",\n \"name\": \"c833939d-30b9-4409-82fd-33d2b22e8826\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T13:10:20.783755056Z\",\n \"endTime\": \"2026-10-02T13:29:28.266052933Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine a7g2b1n4e0w3b0d-autoscale-noaz-xqpdr-gd9vj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4jgj2xjf9rbf--managed/providers/Microsoft.Compute/virtualMachines/a7g2b1n4e0w3b0d-autoscale-noaz-xqpdr-gd9vj to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine a7g2b1n4e0w3b0d-autoscale-noaz-xqpdr-gd9vj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4jgj2xjf9rbf--managed/providers/Microsoft.Compute/virtualMachines/a7g2b1n4e0w3b0d-autoscale-noaz-xqpdr-gd9vj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a14dc0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4jgj2xjf9rbf\" to finish creating: GET https://rp.j6116992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c833939d-30b9-4409-82fd-33d2b22e8826\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c833939d-30b9-4409-82fd-33d2b22e8826\",\n \"name\": \"c833939d-30b9-4409-82fd-33d2b22e8826\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T13:10:20.783755056Z\",\n \"endTime\": \"2026-10-02T13:29:28.266052933Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine a7g2b1n4e0w3b0d-autoscale-noaz-xqpdr-gd9vj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4jgj2xjf9rbf--managed/providers/Microsoft.Compute/virtualMachines/a7g2b1n4e0w3b0d-autoscale-noaz-xqpdr-gd9vj to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine a7g2b1n4e0w3b0d-autoscale-noaz-xqpdr-gd9vj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4jgj2xjf9rbf--managed/providers/Microsoft.Compute/virtualMachines/a7g2b1n4e0w3b0d-autoscale-noaz-xqpdr-gd9vj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105989639176196096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"closed","pr_sha":"7868ef2b26e0bfedc24919742a75cbc3e71f2259","final_merged_sha":"bd1645798d81ff2bb96a575e2cfb1ce745182e60","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T11:53:51Z","started_at":"2026-10-02T11:53:51Z","completed_at":"2026-10-02T14:31:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T14:44:24.054896612Z","timing_metadata_checked_at":"2026-10-02T14:44:24.054896612Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T14:44:24.058248901Z","region_metadata_checked_at":"2026-10-02T14:44:24.058248901Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a5b67132ced837853fcc6a17555d019d80c11e81026c4cc1229513671d3088c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105989639176196096","occurred_at":"2026-10-02T11:53:51Z","failed_at":"e2e","signature_id":"59a6c9ed7c614c1f50c5055404a7b13ac0ce911ce97dad36bf7ce17f21a3947f","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c23e80\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c23e80\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.004s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105989632788271104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7314,"pr_state":"closed","pr_sha":"bbd92b0389c50f2199ac2153f3a2eadd01fee8a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T11:53:50Z","started_at":"2026-10-02T11:53:50Z","completed_at":"2026-10-02T14:23:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T14:33:33.036946788Z","timing_metadata_checked_at":"2026-10-02T14:33:33.036946788Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T14:33:33.039854882Z","region_metadata_checked_at":"2026-10-02T14:33:33.039854882Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0ec1eeeb3be86be3b84fa5f69ab2ab9674ca335ea7b650937d5677a65ab00b1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105989632788271104","occurred_at":"2026-10-02T11:53:50Z","failed_at":"e2e","signature_id":"f69ba6188909b28f74884c51e52b94a32d60de724f0f21d0e969c7c8782b6c2f","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0028843f0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kqrgzxwnxwgd\" to finish creating: GET https://rp.j8271104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b30f413-12b5-4064-9da7-46a76bf0cf7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b30f413-12b5-4064-9da7-46a76bf0cf7b\",\n \"name\": \"7b30f413-12b5-4064-9da7-46a76bf0cf7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T12:59:29.315762309Z\",\n \"endTime\": \"2026-10-02T13:18:36.006994146Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-597b9-prdsc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-597b9-prdsc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0028843f0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kqrgzxwnxwgd\" to finish creating: GET https://rp.j8271104.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b30f413-12b5-4064-9da7-46a76bf0cf7b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7b30f413-12b5-4064-9da7-46a76bf0cf7b\",\n \"name\": \"7b30f413-12b5-4064-9da7-46a76bf0cf7b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T12:59:29.315762309Z\",\n \"endTime\": \"2026-10-02T13:18:36.006994146Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-597b9-prdsc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-597b9-prdsc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105987324146880512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6807,"pr_state":"open","pr_sha":"03e47e849d940901d970997dfa2b85f1ce745e88","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T11:44:40Z","started_at":"2026-10-02T11:44:40Z","completed_at":"2026-10-02T14:19:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T14:33:32.878888422Z","timing_metadata_checked_at":"2026-10-02T14:33:32.878888422Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T14:33:32.881858016Z","region_metadata_checked_at":"2026-10-02T14:33:32.881858016Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"22d31f9b7d7da79784a0fd20a2f833a512fff9b5e25917ed990593cd821cf507","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105987324146880512","occurred_at":"2026-10-02T11:44:40Z","failed_at":"e2e","signature_id":"34274ed735384eaaa0e2b1ae2d0acce3ecf44d677f61c5350015ce1f88a3aa46","test_name":"Customer should create a default cluster and verify KAS is accessible via both public shared ingress and private Swift networking from worker nodes","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:212]: Timed out after 2700.002s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:210 with:\nconsole should return a successful response or redirect, got 503\nExpected\n \u003cint\u003e: ...\nto be \u003c\n \u003cint\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:212]: Timed out after 2700.002s.\npublic ingress should be reachable from outside the VNet\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_public_and_private_kas.go:210 with:\nconsole should return a successful response or redirect, got 503\nExpected\n \u003cint\u003e: ...\nto be \u003c\n \u003cint\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"12b919ef45af23cca2352cbc0e1bd15984998d1d53ba9c82192d5ec6cbeed5c5","failure_pattern":"console should return a successful response or redirect, got 503","search_query":"console should return a successful response or redirect, got 503"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["12b919ef45af23cca2352cbc0e1bd15984998d1d53ba9c82192d5ec6cbeed5c5"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7317/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105978843276775424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7317,"pr_state":"closed","pr_sha":"4289e41ecc652e7e629ca14b20f0f1b510868fa9","final_merged_sha":"7717dd00dcdec243eaed5fb7962fc45f252e6b06","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T11:10:57Z","started_at":"2026-10-02T11:10:57Z","completed_at":"2026-10-02T13:34:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T13:51:52.323847519Z","timing_metadata_checked_at":"2026-10-02T13:51:52.323847519Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T13:51:52.32701741Z","region_metadata_checked_at":"2026-10-02T13:51:52.32701741Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975953711697920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7303,"pr_state":"closed","pr_sha":"479f6d8d8090a97a6f3d5650bf219e2d291e3205","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T10:59:29Z","started_at":"2026-10-02T10:59:29Z","completed_at":"2026-10-02T13:19:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T13:31:19.038409044Z","timing_metadata_checked_at":"2026-10-02T13:31:19.038409044Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T13:31:19.041383534Z","region_metadata_checked_at":"2026-10-02T13:31:19.041383534Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"32d58c256748333c453f40b12f15d6dae30680f45102552e3965c2cd7caaaa3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975953711697920","occurred_at":"2026-10-02T10:59:29Z","failed_at":"alert","signature_id":"ef955f75a2acdcd9710f3aa77383bb014db2593d410f1681e803db0fd48a8c18","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T12:19:46Z\n Ended: 2026-10-02T12:37:47Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jkdxm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-10-02T12:19:46Z\n Ended: 2026-10-02T12:37:46Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jkdxm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"86bf19630c84bafe220feb47fcfee5f84a204e4408b9f5ef28fcb9811b5eec5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975953711697920","occurred_at":"2026-10-02T10:59:29Z","failed_at":"alert","signature_id":"85419bd9eb6fb528fe3ba7717310c0fef873348ff3a1d1a575a7c479071f9143","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T12:24:46Z\n Ended: 2026-10-02T12:39:45Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-jkdxm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"cb7bea04dca3cf6d377be4a5a230d08381db1bcc1c8dde42fa5f1d2f0507ac01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975953711697920","occurred_at":"2026-10-02T10:59:29Z","failed_at":"e2e","signature_id":"6151f3dfe8d1d7e6a99338873294fafd7a249d93b2444cb2b6df005a60d29dcc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-gc94n755ktzv/cluster-candidate-5-0-l6z5kw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001032d80\u003e: \n failed to create HCP cluster cluster-candidate-5-0-l6z5kw: failed waiting for cluster=\"cluster-candidate-5-0-l6z5kw\" in resourcegroup=\"rg-candidate-5-0-gc94n755ktzv\" to finish creating: GET https://rp.j1697920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af1aff57-68d5-48d8-a90a-8e634de5f0cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af1aff57-68d5-48d8-a90a-8e634de5f0cb\",\n \"name\": \"af1aff57-68d5-48d8-a90a-8e634de5f0cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:49:05.164042474Z\",\n \"endTime\": \"2026-10-02T12:18:11.434067357Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-gc94n755ktzv/cluster-candidate-5-0-l6z5kw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001032d80\u003e: \n failed to create HCP cluster cluster-candidate-5-0-l6z5kw: failed waiting for cluster=\"cluster-candidate-5-0-l6z5kw\" in resourcegroup=\"rg-candidate-5-0-gc94n755ktzv\" to finish creating: GET https://rp.j1697920.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af1aff57-68d5-48d8-a90a-8e634de5f0cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af1aff57-68d5-48d8-a90a-8e634de5f0cb\",\n \"name\": \"af1aff57-68d5-48d8-a90a-8e634de5f0cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:49:05.164042474Z\",\n \"endTime\": \"2026-10-02T12:18:11.434067357Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ebda7cf990440ea43cb30f1bf350241e0542d5fa1165904fa1b99d6cde4f5f57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7303/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975953711697920","occurred_at":"2026-10-02T10:59:29Z","failed_at":"alert","signature_id":"acd915c3dfae3e54fbda06bf63f8d1c3e877fc8e6f609f4e70e5b6899c2c7a85","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T12:11:46Z\n Ended: 2026-10-02T12:41:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"3ec768d5-6cf8-4b9c-a960-342894bf7da7\"\n Description: Pod ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-10-02T12:11:46Z\n Ended: 2026-10-02T12:41:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1697920-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.47:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1697920/providers/microsoft.monitor/accounts/hcps-j1697920\", namespace=\"ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"3ec768d5-6cf8-4b9c-a960-342894bf7da7\"\n Description: Pod ocm-arohcpci01-2t6kocm5nlh2bsj6qar6cn2cp8c1dutp-j3g1z4j4b8m5s0b/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975083930488832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7309,"pr_state":"closed","pr_sha":"af8d2c9412c3aa9ef875de263742495a37299cc0","final_merged_sha":"af8d2c9412c3aa9ef875de263742495a37299cc0","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T10:56:01Z","started_at":"2026-10-02T10:56:01Z","completed_at":"2026-10-02T13:13:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T13:31:19.314410159Z","timing_metadata_checked_at":"2026-10-02T13:31:19.314410159Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T13:31:19.31710015Z","region_metadata_checked_at":"2026-10-02T13:31:19.31710015Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"bf8077012adcf431860322eb646f8c1a6e663706e8bfd50855e427d0fba72e4b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975083930488832","occurred_at":"2026-10-02T10:56:01Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T11:59:15Z\n Ended: 2026-10-02T12:14:18Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0488832-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"c22bc865c022c6d8c2d5efbff470b7c90b4f399145dad3d8b4fefec6fbb83982","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975083930488832","occurred_at":"2026-10-02T10:56:01Z","failed_at":"e2e","signature_id":"30cfc0a39a792902039ad9670c2f071e0a56ecab7b20da519e046ac01486e359","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e1940\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-w27q7sgvxjfv\" to finish creating: GET https://rp.j0488832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d3cd18ff-072e-464f-89ca-f4c27910d699\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d3cd18ff-072e-464f-89ca-f4c27910d699\",\n \"name\": \"d3cd18ff-072e-464f-89ca-f4c27910d699\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:50:03.491486549Z\",\n \"endTime\": \"2026-10-02T12:09:10.243715073Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s3a6t6z4u9a9y2v-nodepool-128gib-7vpll-jwmrb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-w27q7sgvxjfv--managed/providers/Microsoft.Compute/virtualMachines/s3a6t6z4u9a9y2v-nodepool-128gib-7vpll-jwmrb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s3a6t6z4u9a9y2v-nodepool-128gib-7vpll-jwmrb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-w27q7sgvxjfv--managed/providers/Microsoft.Compute/virtualMachines/s3a6t6z4u9a9y2v-nodepool-128gib-7vpll-jwmrb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010e1940\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-w27q7sgvxjfv\" to finish creating: GET https://rp.j0488832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d3cd18ff-072e-464f-89ca-f4c27910d699\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d3cd18ff-072e-464f-89ca-f4c27910d699\",\n \"name\": \"d3cd18ff-072e-464f-89ca-f4c27910d699\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:50:03.491486549Z\",\n \"endTime\": \"2026-10-02T12:09:10.243715073Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s3a6t6z4u9a9y2v-nodepool-128gib-7vpll-jwmrb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-w27q7sgvxjfv--managed/providers/Microsoft.Compute/virtualMachines/s3a6t6z4u9a9y2v-nodepool-128gib-7vpll-jwmrb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s3a6t6z4u9a9y2v-nodepool-128gib-7vpll-jwmrb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-w27q7sgvxjfv--managed/providers/Microsoft.Compute/virtualMachines/s3a6t6z4u9a9y2v-nodepool-128gib-7vpll-jwmrb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ed92db6e67f8934cc0f2404411a4f9bfc4f1bdb2bf9e7507096159df25bd83b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7309/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105975083930488832","occurred_at":"2026-10-02T10:56:01Z","failed_at":"e2e","signature_id":"1c44ed3ef9cc244959890b608e75fe48aace2cea67f57efb73c27a002b3dc074","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:247]: failed to create node pool\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f670c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull-create\" in resourcegroup=\"acr-pull-create-tjrqglfwz7jf\" to finish creating: GET https://rp.j0488832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92cbdbab-88df-42a5-95a1-670ebdd455b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92cbdbab-88df-42a5-95a1-670ebdd455b3\",\n \"name\": \"92cbdbab-88df-42a5-95a1-670ebdd455b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:51:54.872760103Z\",\n \"endTime\": \"2026-10-02T12:11:00.580352219Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine acr-pull-create-np-1-rg8t6-jcb2q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-tjrqglfwz7jf--managed/providers/Microsoft.Compute/virtualMachines/acr-pull-create-np-1-rg8t6-jcb2q to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine acr-pull-create-np-1-rg8t6-jcb2q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-tjrqglfwz7jf--managed/providers/Microsoft.Compute/virtualMachines/acr-pull-create-np-1-rg8t6-jcb2q to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:247]: failed to create node pool\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f670c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull-create\" in resourcegroup=\"acr-pull-create-tjrqglfwz7jf\" to finish creating: GET https://rp.j0488832.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92cbdbab-88df-42a5-95a1-670ebdd455b3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/92cbdbab-88df-42a5-95a1-670ebdd455b3\",\n \"name\": \"92cbdbab-88df-42a5-95a1-670ebdd455b3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:51:54.872760103Z\",\n \"endTime\": \"2026-10-02T12:11:00.580352219Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine acr-pull-create-np-1-rg8t6-jcb2q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-tjrqglfwz7jf--managed/providers/Microsoft.Compute/virtualMachines/acr-pull-create-np-1-rg8t6-jcb2q to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine acr-pull-create-np-1-rg8t6-jcb2q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-tjrqglfwz7jf--managed/providers/Microsoft.Compute/virtualMachines/acr-pull-create-np-1-rg8t6-jcb2q to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"closed","pr_sha":"57fcaa2257a16269476e6901168ab5f7eb65d0a6","final_merged_sha":"ea2df83db19482a983a29965803a71b1f8158ab7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T10:09:31Z","started_at":"2026-10-02T10:09:31Z","completed_at":"2026-10-02T12:51:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T13:10:20.865398948Z","timing_metadata_checked_at":"2026-10-02T13:10:20.865398948Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T13:10:20.869020635Z","region_metadata_checked_at":"2026-10-02T13:10:20.869020635Z"},"failed_at":["alert","e2e"],"failed_test_count":12,"occurrences":[{"row_id":"1016b78006f24f5a04292ad33af214b15047bba150ccc100619506bc6f85fce7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"3a9b1d0a1756d4352283ba3b63e3d886955fa9c5c336253e1b019206952a5507","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002032960\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-s47ln9cbqhqp\" to finish creating: GET https://rp.j7658624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b17a89af-b196-4b23-bd98-9d9ab5a5f12a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b17a89af-b196-4b23-bd98-9d9ab5a5f12a\",\n \"name\": \"b17a89af-b196-4b23-bd98-9d9ab5a5f12a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:09:22.074259734Z\",\n \"endTime\": \"2026-10-02T11:28:22.963531957Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-kcxkl-4xf9f: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-s47ln9cbqhqp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-kcxkl-4xf9f to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-kcxkl-4xf9f: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-s47ln9cbqhqp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-kcxkl-4xf9f to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002032960\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-s47ln9cbqhqp\" to finish creating: GET https://rp.j7658624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b17a89af-b196-4b23-bd98-9d9ab5a5f12a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b17a89af-b196-4b23-bd98-9d9ab5a5f12a\",\n \"name\": \"b17a89af-b196-4b23-bd98-9d9ab5a5f12a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:09:22.074259734Z\",\n \"endTime\": \"2026-10-02T11:28:22.963531957Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-kcxkl-4xf9f: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-s47ln9cbqhqp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-kcxkl-4xf9f to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-kcxkl-4xf9f: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-s47ln9cbqhqp--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-kcxkl-4xf9f to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1998209cd60bf2c9d31d48f646734939d79a7f7b352a9d0fb8674026f952de0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"9b85a307c626519801ac209425f1debdf469e99c3d127b2e7e94b3fca1fbcb53","test_name":"Customer should upgrade control plane minor version when pods cannot be created from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.009s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-dg8wqw\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0015c3050\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade_blocked_pod_creation.go:240]: Timed out after 2700.009s.\ncontrol plane did not reach 5.1 on cluster \"cp-ystream-deny-pods-5-1-dg8wqw\" with deny-pod ValidatingAdmissionPolicy\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0015c3050\u003e: \n kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"db42d3a8bb432b55cd45f2d44660dfafa02906a8443476f401d5f2c067022daa","failure_pattern":"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"1d5b37702d4ce1d882f702b4d3a4d0c7e91cf3483c7994dccbdc30713f72fa0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"247f2f7e907043573a578d118d78f6c6f00500e34fc61faec7dbd664db892fcf","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000dc15d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000dc15d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"1e8a1c0adc16f06a7e3c569b848dcdf89423854890ff95f87bda33c180fd5462","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"8dbea7b9ec3df592e7098abc9c237a0026e2477ed890812177c981f0246ac466","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00063e600\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-snc8bz4l2lpm\" to finish creating: GET https://rp.j7658624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/377e91a7-9fb9-4364-a4c3-94d301101999\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/377e91a7-9fb9-4364-a4c3-94d301101999\",\n \"name\": \"377e91a7-9fb9-4364-a4c3-94d301101999\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:41:21.075539741Z\",\n \"endTime\": \"2026-10-02T12:00:23.993693871Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-snc8bz4l2lpm--managed/providers/Microsoft.Compute/virtualMachines/k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-snc8bz4l2lpm--managed/providers/Microsoft.Compute/virtualMachines/k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00063e600\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-snc8bz4l2lpm\" to finish creating: GET https://rp.j7658624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/377e91a7-9fb9-4364-a4c3-94d301101999\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/377e91a7-9fb9-4364-a4c3-94d301101999\",\n \"name\": \"377e91a7-9fb9-4364-a4c3-94d301101999\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:41:21.075539741Z\",\n \"endTime\": \"2026-10-02T12:00:23.993693871Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-snc8bz4l2lpm--managed/providers/Microsoft.Compute/virtualMachines/k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-snc8bz4l2lpm--managed/providers/Microsoft.Compute/virtualMachines/k8j2o5c1s9g3f7l-np-delete-main-gpk8z-64bzj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-46hf4d98wdxd\" to finish creating: GET https://rp"}},{"row_id":"3ad40a21fd3c65e44509af8c0b01724131d3f56c5eb01ab0dd7b47af1e29611d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"alert","signature_id":"816db2d3fb1d50b6391b3d4ae23ae87f6b66058b291be9d7e3fc5c359fca5e80","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T11:16:17Z\n Ended: 2026-10-02T11:26:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7658624-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t6k02kauka61vjc7pvhhdhndnsmtknd-ea-list6nlmtx\", pod=\"kube-apiserver-67bb546754-b454t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t6k02kauka61vjc7pvhhdhndnsmtknd-ea-list6nlmtx/kube-apiserver-67bb546754-b454t has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"5c8a6962ce2413646b9cefdf864cc5691c319a0e325826ef2dae49fe272ac86d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"d285c70301184b9a72cbfbcf497f8bb394dd1683ff0d5b4206fc2a65203f44fb","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:193]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0015c66c0\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nqllmtrxwh84\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-nqllmtrxwh84, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:193]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0015c66c0\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nqllmtrxwh84\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-nqllmtrxwh84, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-nqllmtrxwh84\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-nqllmtrxwh84, error: context deadline exceeded"}},{"row_id":"7945d468b3cf19ea36a36b35af2e05fa357367dab89cfae580239626fe49dc71","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"e903e6214fc854d426efd67d556189c90371a5d9fd33b4c1bbc8c31c92089a9a","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:416]: failed to poll cluster \"cidr-connectivity-test\" authorized CIDRs update to completion (timeout '10.000000' minutes)\nUnexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:416]: failed to poll cluster \"cidr-connectivity-test\" authorized CIDRs update to completion (timeout '10.000000' minutes)\nUnexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"17e5892e8c32879055400ab93a946ce79ad6cc684588825e1e1a488695d06f44","failure_pattern":"failed to poll cluster \"\u003ccluster\u003e\" authorized CIDRs update to completion (timeout '10.000000' minutes)","search_query":"context deadline exceeded"}},{"row_id":"996f2bab5f5eba1fed7863e22ed28c0c064fcde596663920da3952a97418ca82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"c4fa5c26ee982b83676c3b7397954d31e709a456e87dee9158b60d5d5dfcd62f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-j4v4c9\" for cluster \"cluster-candidate-4-20-j4v4c9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001238cc0\u003e: \n failed to create NodePool np-j4v4c9: failed waiting for nodepool=\"np-j4v4c9\" for cluster \"cluster-candidate-4-20-j4v4c9\" in resourcegroup=\"rg-candidate-4-20-ck8mc4szhlcn\" to finish creating: GET https://rp.j7658624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0d415e4-e138-4042-a3b0-d8c32121b5e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0d415e4-e138-4042-a3b0-d8c32121b5e0\",\n \"name\": \"f0d415e4-e138-4042-a3b0-d8c32121b5e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:08:30.228624897Z\",\n \"endTime\": \"2026-10-02T11:27:32.877879167Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4v6j6x4k0f0u5k-np-j4v4c9-h2pws-dlbbq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ck8mc4szhlcn-candidate-j4v4c9--managed/providers/Microsoft.Compute/virtualMachines/m4v6j6x4k0f0u5k-np-j4v4c9-h2pws-dlbbq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4v6j6x4k0f0u5k-np-j4v4c9-h2pws-dlbbq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ck8mc4szhlcn-candidate-j4v4c9--managed/providers/Microsoft.Compute/virtualMachines/m4v6j6x4k0f0u5k-np-j4v4c9-h2pws-dlbbq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-j4v4c9\" for cluster \"cluster-candidate-4-20-j4v4c9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001238cc0\u003e: \n failed to create NodePool np-j4v4c9: failed waiting for nodepool=\"np-j4v4c9\" for cluster \"cluster-candidate-4-20-j4v4c9\" in resourcegroup=\"rg-candidate-4-20-ck8mc4szhlcn\" to finish creating: GET https://rp.j7658624.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0d415e4-e138-4042-a3b0-d8c32121b5e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f0d415e4-e138-4042-a3b0-d8c32121b5e0\",\n \"name\": \"f0d415e4-e138-4042-a3b0-d8c32121b5e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T11:08:30.228624897Z\",\n \"endTime\": \"2026-10-02T11:27:32.877879167Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4v6j6x4k0f0u5k-np-j4v4c9-h2pws-dlbbq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ck8mc4szhlcn-candidate-j4v4c9--managed/providers/Microsoft.Compute/virtualMachines/m4v6j6x4k0f0u5k-np-j4v4c9-h2pws-dlbbq to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4v6j6x4k0f0u5k-np-j4v4c9-h2pws-dlbbq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-ck8mc4szhlcn-candidate-j4v4c9--managed/providers/Microsoft.Compute/virtualMachines/m4v6j6x4k0f0u5k-np-j4v4c9-h2pws-dlbbq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a87c2fb0ca0e1d5085620a712cf5a48a156beff95104b317107f6155066121a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"61300846ff2ab19ece6d91bcf7c616bc6175f4c4f2bfefccd840a12b6302252a","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0011d8120\u003e: \n failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9h5ws9hlzq4p\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9h5ws9hlzq4p, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0011d8120\u003e: \n failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9h5ws9hlzq4p\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9h5ws9hlzq4p, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2992815bad38a5769dd245becb8213e9b3a7e85882b8db6a819eb56049478a21","failure_pattern":"UpdateHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-9h5ws9hlzq4p\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-9h5ws9hlzq4p, error: context deadline exceeded"}},{"row_id":"a88de9a536ef8d1103b626223e566de11f65958dab825b6d5af2c6c03378d53d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"1c150067c2543dc56eca317c3be63944fbdc62dfd7d1e1aae30701f1f3c48892","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0013d63c0\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d4f6lxz4s2dz\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-d4f6lxz4s2dz, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0013d63c0\u003e: \n failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d4f6lxz4s2dz\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-d4f6lxz4s2dz, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0d75e828705d5e967c897d0839b4327d675cfc294e87dead6ef776571e28083d","failure_pattern":"UpdateHCPCluster20251223 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-d4f6lxz4s2dz\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20251223 for cluster idms-e2e-hcp-cluster in resource group idms-d4f6lxz4s2dz, error: context deadline exceeded"}},{"row_id":"d08f7e58bba0e768ed7b454144ad28eb72902ca83448de3bf04ade8376aa80e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"b721d7faa183dc16fbcbacdfc908d1c3712e92ed9f08208bc9d61279db4bba79","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:369]: failed to update containerRegistry via PATCH\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000fa8690\u003e: \n failed waiting for hcpCluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-ljhpblch2468\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20261001 for cluster acr-pull-create in resource group acr-pull-create-ljhpblch2468, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:369]: failed to update containerRegistry via PATCH\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000fa8690\u003e: \n failed waiting for hcpCluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-ljhpblch2468\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20261001 for cluster acr-pull-create in resource group acr-pull-create-ljhpblch2468, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bcb79b4f02a40cf142723d935e5b38797b059e620022fb96857e13363ac752f3","failure_pattern":"UpdateHCPCluster20261001 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-ljhpblch2468\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster20261001 for cluster acr-pull-create in resource group acr-pull-create-ljhpblch2468, error: context deadline exceeded"}},{"row_id":"f9ba8552eb73c1a8967f1120454585b40ec5c1e97164afe908a17fc7ddb31fb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105963382757658624","occurred_at":"2026-10-02T10:09:31Z","failed_at":"e2e","signature_id":"da4948d61ab54e112557ad6fe48bbeb3135dc01594a49200da12544749a36fa1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:282]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:282]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n \u003ccontext.deadlineExceededError\u003e: \n context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e0fae1df9008b1d12ccc10847571ae065f9b820f22326abb57a797bc01d29b98","failure_pattern":"failed to update cluster \"\u003ccluster\u003e\" authorized CIDRs to include test runner IP","search_query":"context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["0d75e828705d5e967c897d0839b4327d675cfc294e87dead6ef776571e28083d","17e5892e8c32879055400ab93a946ce79ad6cc684588825e1e1a488695d06f44","2992815bad38a5769dd245becb8213e9b3a7e85882b8db6a819eb56049478a21","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","bcb79b4f02a40cf142723d935e5b38797b059e620022fb96857e13363ac752f3","db42d3a8bb432b55cd45f2d44660dfafa02906a8443476f401d5f2c067022daa","e0fae1df9008b1d12ccc10847571ae065f9b820f22326abb57a797bc01d29b98","f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6735/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105958205317189632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6735,"pr_state":"closed","pr_sha":"7d780d625adde0e1b1135185e59622f77aaa44d5","final_merged_sha":"7d780d625adde0e1b1135185e59622f77aaa44d5","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T09:48:57Z","started_at":"2026-10-02T09:48:57Z","completed_at":"2026-10-02T12:12:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T12:28:53.824993935Z","timing_metadata_checked_at":"2026-10-02T12:28:53.824993935Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T12:28:53.828583924Z","region_metadata_checked_at":"2026-10-02T12:28:53.828583924Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7292/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105954097692676096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7292,"pr_state":"open","pr_sha":"aecbf5f4e34005d22c03a5bdef7e284ae13874e1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T09:32:38Z","started_at":"2026-10-02T09:32:38Z","completed_at":"2026-10-02T11:53:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T12:07:54.520003825Z","timing_metadata_checked_at":"2026-10-02T12:07:54.520003825Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T12:07:54.523681213Z","region_metadata_checked_at":"2026-10-02T12:07:54.523681213Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7316/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105935535393476608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7316,"pr_state":"closed","pr_sha":"02d158fcf7d339f3c8882674e67f1b445d8fb40d","final_merged_sha":"02d158fcf7d339f3c8882674e67f1b445d8fb40d","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T08:18:52Z","started_at":"2026-10-02T08:18:52Z","completed_at":"2026-10-02T10:35:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T10:53:19.160574524Z","timing_metadata_checked_at":"2026-10-02T10:53:19.160574524Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T10:53:19.163702913Z","region_metadata_checked_at":"2026-10-02T10:53:19.163702913Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105933227649667072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7293,"pr_state":"closed","pr_sha":"0a9ebe5cc081ab4b79193a8e353f5560596c2406","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T08:09:42Z","started_at":"2026-10-02T08:09:42Z","completed_at":"2026-10-02T10:28:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T10:42:20.783134828Z","timing_metadata_checked_at":"2026-10-02T10:42:20.783134828Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T10:42:20.790657304Z","region_metadata_checked_at":"2026-10-02T10:42:20.790657304Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"2131d7f59500d5e4ec53a798c3b42075ea8e23e2d7dd327a96006670327ef9d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105933227649667072","occurred_at":"2026-10-02T08:09:42Z","failed_at":"alert","signature_id":"1f3b34a45cf8a5ee02900fdfe7d8f46291016a5dd0e2aac524d7b4e070d18d47","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T09:12:36Z\n Ended: 2026-10-02T09:27:35Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9667072-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-10-02T09:12:36Z\n Ended: 2026-10-02T09:30:37Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9667072-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n State: Resolved\n Started: 2026-10-02T09:12:36Z\n Ended: 2026-10-02T09:28:36Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9667072-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105929411579613184","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T07:54:32Z","started_at":"2026-10-02T07:54:32Z","completed_at":"2026-10-02T10:21:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T10:32:01.113314509Z","timing_metadata_checked_at":"2026-10-02T10:32:01.113314509Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T10:32:01.116849201Z","region_metadata_checked_at":"2026-10-02T10:32:01.116849201Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105916967100616704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"closed","pr_sha":"7868ef2b26e0bfedc24919742a75cbc3e71f2259","final_merged_sha":"bd1645798d81ff2bb96a575e2cfb1ce745182e60","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T07:05:05Z","started_at":"2026-10-02T07:05:05Z","completed_at":"2026-10-02T09:24:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T09:40:02.879839651Z","timing_metadata_checked_at":"2026-10-02T09:40:02.879839651Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T09:40:02.88320994Z","region_metadata_checked_at":"2026-10-02T09:40:02.88320994Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"031f6917e61dcd4399d24a5729a16d3dee2370f9f754b18b5eb77e7c83ac6bcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105916967100616704","occurred_at":"2026-10-02T07:05:05Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-10-02T08:10:58Z\n Ended: 2026-10-02T08:29:58Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0616704-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"b6204b8ca61f39399745539ba90f9fcd24272fc8008ec16689c7b42b726ecb7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105916967100616704","occurred_at":"2026-10-02T07:05:05Z","failed_at":"e2e","signature_id":"aa94cf88bb79706f28a4d806b8b9860fe56b6477bb4b62bfc09ab3f7354f8d08","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b84f00\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2vzgl45q9ll4\" to finish creating: GET https://rp.j0616704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/992f2fbe-f68f-43d2-ae3a-903d3d70bb71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/992f2fbe-f68f-43d2-ae3a-903d3d70bb71\",\n \"name\": \"992f2fbe-f68f-43d2-ae3a-903d3d70bb71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:27:53.917230714Z\",\n \"endTime\": \"2026-10-02T08:46:59.056787427Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4k0b7t8n9c5i1g-np-1-4f2zv-4fztr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4k0b7t8n9c5i1g-np-1-4f2zv-4fztr: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b84f00\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2vzgl45q9ll4\" to finish creating: GET https://rp.j0616704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/992f2fbe-f68f-43d2-ae3a-903d3d70bb71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/992f2fbe-f68f-43d2-ae3a-903d3d70bb71\",\n \"name\": \"992f2fbe-f68f-43d2-ae3a-903d3d70bb71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:27:53.917230714Z\",\n \"endTime\": \"2026-10-02T08:46:59.056787427Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4k0b7t8n9c5i1g-np-1-4f2zv-4fztr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4k0b7t8n9c5i1g-np-1-4f2zv-4fztr: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","fd0407baf94133db7fa1758b83668acc74fdd3ca69dbe3150c9481e216c634ef"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105899086866550784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"open","pr_sha":"1495f07f761e3c094aa60549d95c4cf2cf1748e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T05:54:02Z","started_at":"2026-10-02T05:54:02Z","completed_at":"2026-10-02T08:09:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T08:26:56.686996356Z","timing_metadata_checked_at":"2026-10-02T08:26:56.686996356Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T08:26:56.690377744Z","region_metadata_checked_at":"2026-10-02T08:26:56.690377744Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a8e0c43185a78b7dfd5a6e5bd7eb1616ba9bf79f480301f94169e9410abf73fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105899086866550784","occurred_at":"2026-10-02T05:54:02Z","failed_at":"e2e","signature_id":"452eed4657b73553008ac22f54a6f1e9229b5abe0497e58b51b804e7a82c75ad","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7ctmvh\" for cluster \"cluster-candidate-4-23-7ctmvh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb30a0\u003e: \n failed to create NodePool np-7ctmvh: failed waiting for nodepool=\"np-7ctmvh\" for cluster \"cluster-candidate-4-23-7ctmvh\" in resourcegroup=\"rg-candidate-4-23-ztlck247gbc7\" to finish creating: GET https://rp.j6550784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28de2827-3d1e-473f-9ac1-7443442e5482\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28de2827-3d1e-473f-9ac1-7443442e5482\",\n \"name\": \"28de2827-3d1e-473f-9ac1-7443442e5482\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T06:54:46.375852677Z\",\n \"endTime\": \"2026-10-02T07:13:49.710009064Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r9l8y1w3n6j7l5a-np-7ctmvh-v8j29-2gnt5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-ztlck247gbc7-candidate-7ctmvh--managed/providers/Microsoft.Compute/virtualMachines/r9l8y1w3n6j7l5a-np-7ctmvh-v8j29-2gnt5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r9l8y1w3n6j7l5a-np-7ctmvh-v8j29-2gnt5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-ztlck247gbc7-candidate-7ctmvh--managed/providers/Microsoft.Compute/virtualMachines/r9l8y1w3n6j7l5a-np-7ctmvh-v8j29-2gnt5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7ctmvh\" for cluster \"cluster-candidate-4-23-7ctmvh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb30a0\u003e: \n failed to create NodePool np-7ctmvh: failed waiting for nodepool=\"np-7ctmvh\" for cluster \"cluster-candidate-4-23-7ctmvh\" in resourcegroup=\"rg-candidate-4-23-ztlck247gbc7\" to finish creating: GET https://rp.j6550784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28de2827-3d1e-473f-9ac1-7443442e5482\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/28de2827-3d1e-473f-9ac1-7443442e5482\",\n \"name\": \"28de2827-3d1e-473f-9ac1-7443442e5482\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T06:54:46.375852677Z\",\n \"endTime\": \"2026-10-02T07:13:49.710009064Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r9l8y1w3n6j7l5a-np-7ctmvh-v8j29-2gnt5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-ztlck247gbc7-candidate-7ctmvh--managed/providers/Microsoft.Compute/virtualMachines/r9l8y1w3n6j7l5a-np-7ctmvh-v8j29-2gnt5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r9l8y1w3n6j7l5a-np-7ctmvh-v8j29-2gnt5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-ztlck247gbc7-candidate-7ctmvh--managed/providers/Microsoft.Compute/virtualMachines/r9l8y1w3n6j7l5a-np-7ctmvh-v8j29-2gnt5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105895928454975488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"4121ec5845c2ca0150e25dfeea026e610137763b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T05:41:29Z","started_at":"2026-10-02T05:41:29Z","completed_at":"2026-10-02T08:03:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T08:16:45.134726453Z","timing_metadata_checked_at":"2026-10-02T08:16:45.134726453Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T08:16:45.140037235Z","region_metadata_checked_at":"2026-10-02T08:16:45.140037235Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7307/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105886309867327488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7307,"pr_state":"open","pr_sha":"c4679ff1add0334f705c136a588ca63e81f8c980","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T05:03:16Z","started_at":"2026-10-02T05:03:16Z","completed_at":"2026-10-02T07:23:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T07:34:19.605420099Z","timing_metadata_checked_at":"2026-10-02T07:34:19.605420099Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T07:34:19.609023388Z","region_metadata_checked_at":"2026-10-02T07:34:19.609023388Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105865001846706176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"4121ec5845c2ca0150e25dfeea026e610137763b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T03:38:36Z","started_at":"2026-10-02T03:38:36Z","completed_at":"2026-10-02T04:30:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T04:36:25.245937801Z","timing_metadata_checked_at":"2026-10-02T04:36:25.245937801Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T04:36:25.248901092Z","region_metadata_checked_at":"2026-10-02T04:36:25.248901092Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"88aa5759ffed4f19f06ea705471bd950a5334135c8dc0dc3440f2fc0a20c69fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105865001846706176","occurred_at":"2026-10-02T03:38:36Z","failed_at":"provision","signature_id":"65d9370e60af642e906378ad8145d81c4e331f8977d3894f60b0c564c92ed6e5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-02T04:18:24.964Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-02T04:18:25.604Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 values=\"map[azure:map[clientId:3a6b5dce-c503-483f-868b-3656736645ed cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-ci01-maestro-j6706176.westus3-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j6706176-mgmt-1 credsKeyVault:map[name:ah-ci01-mg-j6706176-1 secret:hcp-underlay-j6706176-mgmt-1] deployment:map[zoneCount:3] glog_v:2 image:map[digest:sha256:9af415e086251e94f471e03844ce768bed6493653b6cabdaff68acf99573679d registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:1 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:3a8316d42c6491dbc15f2cf59985901365c7c402d3a0649abe695a4a6554654e registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-10-02T04:18:25.604Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:18:26.032Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 namespace=maestro\ntime=2026-10-02T04:18:26.347Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-02T04:18:26.347Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:18:26.470Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:23:32.794Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-02T04:23:32.795Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0c8f285ebf7a77151b5c5324a77de4ff3bf14831af7586b962d27c5eacb2a8e6","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0c8f285ebf7a77151b5c5324a77de4ff3bf14831af7586b962d27c5eacb2a8e6"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860632891887616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7314,"pr_state":"closed","pr_sha":"bbd92b0389c50f2199ac2153f3a2eadd01fee8a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T03:21:14Z","started_at":"2026-10-02T03:21:14Z","completed_at":"2026-10-02T05:40:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T05:59:52.974116261Z","timing_metadata_checked_at":"2026-10-02T05:59:52.974116261Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T05:59:52.976695153Z","region_metadata_checked_at":"2026-10-02T05:59:52.976695153Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c296105b64d2300faa371a84ba608cd00acb01cd4533c17edf81f4efe93ff5bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7314/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860632891887616","occurred_at":"2026-10-02T03:21:14Z","failed_at":"e2e","signature_id":"79f30e35241b4c02e2ba6c027a86d8d186f0dd2acc5e0e41d5fd756e33a11c74","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9d1e0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-z847fd\" in resourcegroup=\"rg-np-version-upgrade-z847fd-9d7c47wqwcb4\" to finish creating: GET https://rp.j1887616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b15e942-32f6-4100-897e-aeb4bb2b3983\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b15e942-32f6-4100-897e-aeb4bb2b3983\",\n \"name\": \"6b15e942-32f6-4100-897e-aeb4bb2b3983\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T04:46:32.427711572Z\",\n \"endTime\": \"2026-10-02T05:05:35.48212253Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0m2l9l1f2b4p9g-npupgrade-4-20-5rnx4-mqdsr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z847fd-9d7c47wqwcb4-np-upgrade-z8-2e598309/providers/Microsoft.Compute/virtualMachines/q0m2l9l1f2b4p9g-npupgrade-4-20-5rnx4-mqdsr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0m2l9l1f2b4p9g-npupgrade-4-20-5rnx4-mqdsr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z847fd-9d7c47wqwcb4-np-upgrade-z8-2e598309/providers/Microsoft.Compute/virtualMachines/q0m2l9l1f2b4p9g-npupgrade-4-20-5rnx4-mqdsr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e9d1e0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-z847fd\" in resourcegroup=\"rg-np-version-upgrade-z847fd-9d7c47wqwcb4\" to finish creating: GET https://rp.j1887616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b15e942-32f6-4100-897e-aeb4bb2b3983\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6b15e942-32f6-4100-897e-aeb4bb2b3983\",\n \"name\": \"6b15e942-32f6-4100-897e-aeb4bb2b3983\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T04:46:32.427711572Z\",\n \"endTime\": \"2026-10-02T05:05:35.48212253Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q0m2l9l1f2b4p9g-npupgrade-4-20-5rnx4-mqdsr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z847fd-9d7c47wqwcb4-np-upgrade-z8-2e598309/providers/Microsoft.Compute/virtualMachines/q0m2l9l1f2b4p9g-npupgrade-4-20-5rnx4-mqdsr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q0m2l9l1f2b4p9g-npupgrade-4-20-5rnx4-mqdsr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z847fd-9d7c47wqwcb4-np-upgrade-z8-2e598309/providers/Microsoft.Compute/virtualMachines/q0m2l9l1f2b4p9g-npupgrade-4-20-5rnx4-mqdsr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860601363304448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"open","pr_sha":"1495f07f761e3c094aa60549d95c4cf2cf1748e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T03:21:06Z","started_at":"2026-10-02T03:21:06Z","completed_at":"2026-10-02T04:16:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T04:26:07.173919929Z","timing_metadata_checked_at":"2026-10-02T04:26:07.173919929Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T04:26:07.179800411Z","region_metadata_checked_at":"2026-10-02T04:26:07.179800411Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"56d8d4c33960cb87a5a469eed9f745565e52abebe2e4cee0e630dc49423930b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860601363304448","occurred_at":"2026-10-02T03:21:06Z","failed_at":"alert","signature_id":"292f546bc904e43ecc8422922e08247071ec4dff8f7132975575642549b9e4ef","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-wzq9v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/prometheus-operator-5d45794ff-wzq9v has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-qj4ln\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/prometheus-operator-5d45794ff-qj4ln has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-w44fw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-w44fw has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"arohcp-monitor-kube-state-metrics-79b88f5699-66rf9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/arohcp-monitor-kube-state-metrics-79b88f5699-66rf9 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-10-02T04:11:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3304448-svc\", component=\"kubernetes-infrastructure\", namespace=\"prometheus\", pod=\"prometheus-operator-5d45794ff-4p6tc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod prometheus/prometheus-operator-5d45794ff-4p6tc has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"a2ae860d49e63ddec08514164c5e048d7fb25933145a81beffc0f2dedd7d93d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860601363304448","occurred_at":"2026-10-02T03:21:06Z","failed_at":"provision","signature_id":"4ab9e4e3b3cde7399dc6f8c4bc34053ae4761a23ab44459a95beb2590036596e","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Maestro.Agent/management/deploy (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-10-02T04:04:12.931Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-10-02T04:04:13.290Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 values=\"map[azure:map[clientId:ad930020-a0d3-4a84-b682-06665523a404 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] broker:map[host:arohcp-ci01-maestro-j3304448.canadacentral-1.ts.eventgrid.azure.net port:8883] consumerName:hcp-underlay-j3304448-mgmt-1 credsKeyVault:map[name:ah-ci01-mg-j3304448-1 secret:hcp-underlay-j3304448-mgmt-1] deployment:map[zoneCount:3] glog_v:2 image:map[digest:sha256:9af415e086251e94f471e03844ce768bed6493653b6cabdaff68acf99573679d registry:arohcpsvcdev.azurecr.io repository:redhat-user-workloads/maestro-rhtap-tenant/maestro/maestro] installAppliedManifestWorkCRD:false replicas:1 serviceAccountName:maestro sideCar:map[image:map[digest:sha256:3a8316d42c6491dbc15f2cf59985901365c7c402d3a0649abe695a4a6554654e registry:mcr.microsoft.com repository:azurelinux/base/nginx]]]\"\ntime=2026-10-02T04:04:13.290Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:04:13.437Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 namespace=maestro\ntime=2026-10-02T04:04:13.594Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 dryRun=false\ntime=2026-10-02T04:04:13.594Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:04:13.638Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1\ntime=2026-10-02T04:09:17.935Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 err=\"resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-10-02T04:09:17.936Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Maestro.Agent resourceGroup=management step=deploy stamp=1 err=\"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0c8f285ebf7a77151b5c5324a77de4ff3bf14831af7586b962d27c5eacb2a8e6","failure_pattern":"failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"stamp 1: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/maestro/maestro-agent not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["0c8f285ebf7a77151b5c5324a77de4ff3bf14831af7586b962d27c5eacb2a8e6","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860557423775744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6775,"pr_state":"closed","pr_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","final_merged_sha":"02cba68fff12e77be3d865d9e0d24a307718ee82","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T03:20:56Z","started_at":"2026-10-02T03:20:56Z","completed_at":"2026-10-02T05:46:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T05:59:52.681241951Z","timing_metadata_checked_at":"2026-10-02T05:59:52.681241951Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T05:59:52.684587141Z","region_metadata_checked_at":"2026-10-02T05:59:52.684587141Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9bafa6af57d6afa50f0b23746e38411c986b335ba66dc05aa06d106fe9c91065","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6775/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105860557423775744","occurred_at":"2026-10-02T03:20:56Z","failed_at":"e2e","signature_id":"c762968dfb2da1a7708f604dd85763b8611979abccdf3d8a74f487a345b560c1","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90e60\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-gxq8mc\" in resourcegroup=\"rg-np-skip-minor-gxq8mc-kwv67gxfnh4r\" to finish creating: GET https://rp.j3775744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd38511-4f3f-4db5-ba8c-93507fe57cde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd38511-4f3f-4db5-ba8c-93507fe57cde\",\n \"name\": \"5dd38511-4f3f-4db5-ba8c-93507fe57cde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T04:52:30.925064823Z\",\n \"endTime\": \"2026-10-02T05:11:35.851064932Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v0b4f8d8t1c5i8g-nps-420-422-7mmqj-6nqb7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-gxq8mc-kwv67gxfnh4r-np-sm-gxq8mc--managed/providers/Microsoft.Compute/virtualMachines/v0b4f8d8t1c5i8g-nps-420-422-7mmqj-6nqb7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v0b4f8d8t1c5i8g-nps-420-422-7mmqj-6nqb7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-gxq8mc-kwv67gxfnh4r-np-sm-gxq8mc--managed/providers/Microsoft.Compute/virtualMachines/v0b4f8d8t1c5i8g-nps-420-422-7mmqj-6nqb7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.41\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90e60\u003e: \n failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-gxq8mc\" in resourcegroup=\"rg-np-skip-minor-gxq8mc-kwv67gxfnh4r\" to finish creating: GET https://rp.j3775744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd38511-4f3f-4db5-ba8c-93507fe57cde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5dd38511-4f3f-4db5-ba8c-93507fe57cde\",\n \"name\": \"5dd38511-4f3f-4db5-ba8c-93507fe57cde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T04:52:30.925064823Z\",\n \"endTime\": \"2026-10-02T05:11:35.851064932Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v0b4f8d8t1c5i8g-nps-420-422-7mmqj-6nqb7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-gxq8mc-kwv67gxfnh4r-np-sm-gxq8mc--managed/providers/Microsoft.Compute/virtualMachines/v0b4f8d8t1c5i8g-nps-420-422-7mmqj-6nqb7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v0b4f8d8t1c5i8g-nps-420-422-7mmqj-6nqb7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-gxq8mc-kwv67gxfnh4r-np-sm-gxq8mc--managed/providers/Microsoft.Compute/virtualMachines/v0b4f8d8t1c5i8g-nps-420-422-7mmqj-6nqb7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7260/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105859844991881216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7260,"pr_state":"closed","pr_sha":"e2029a06a51da6cdf2ed5b5ef1c620dcc9cda152","final_merged_sha":"e2029a06a51da6cdf2ed5b5ef1c620dcc9cda152","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T03:18:06Z","started_at":"2026-10-02T03:18:06Z","completed_at":"2026-10-02T05:45:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T05:59:52.825320413Z","timing_metadata_checked_at":"2026-10-02T05:59:52.825320413Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T05:59:52.829116801Z","region_metadata_checked_at":"2026-10-02T05:59:52.829116801Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105834562247462912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"5d263fda90bbaad9782b3a3ce393cbf1e6eb03eb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-02T01:37:38Z","started_at":"2026-10-02T01:37:38Z","completed_at":"2026-10-02T03:52:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T03:54:53.549558072Z","timing_metadata_checked_at":"2026-10-02T03:54:53.549558072Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T03:54:53.552241664Z","region_metadata_checked_at":"2026-10-02T03:54:53.552241664Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105830212750020608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7293,"pr_state":"closed","pr_sha":"0a9ebe5cc081ab4b79193a8e353f5560596c2406","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T01:20:21Z","started_at":"2026-10-02T01:20:21Z","completed_at":"2026-10-02T03:23:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T03:34:06.523409886Z","timing_metadata_checked_at":"2026-10-02T03:34:06.523409886Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T03:34:06.526183478Z","region_metadata_checked_at":"2026-10-02T03:34:06.526183478Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8a925c2714bc12a437cc5aa82b8c30c1df99fac5aa37a9024d42b98b83399022","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7293/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105830212750020608","occurred_at":"2026-10-02T01:20:21Z","failed_at":"e2e","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105811761943810048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7105,"pr_state":"open","pr_sha":"1495f07f761e3c094aa60549d95c4cf2cf1748e9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-02T00:07:02Z","started_at":"2026-10-02T00:07:02Z","completed_at":"2026-10-02T02:31:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T02:52:08.330700464Z","timing_metadata_checked_at":"2026-10-02T02:52:08.330700464Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T02:52:08.333740854Z","region_metadata_checked_at":"2026-10-02T02:52:08.333740854Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2e67da79c411289b31b869d79a7fd66b23a07e6f53cb37d932d919642a923bd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105811761943810048","occurred_at":"2026-10-02T00:07:02Z","failed_at":"e2e","signature_id":"e68aa4f1fd5584a77fb4d1bf176cf9e57b599009a951da1f3d48eac028964cfb","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fef1c0\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-svbr9q\" in resourcegroup=\"rg-np-downgrade-svbr9q-kvqmb7zbtpsq\" to finish creating: GET https://rp.j3810048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/556c3a8a-eee8-47cd-95b3-aed05e7e4d22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/556c3a8a-eee8-47cd-95b3-aed05e7e4d22\",\n \"name\": \"556c3a8a-eee8-47cd-95b3-aed05e7e4d22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:38:44.187977267Z\",\n \"endTime\": \"2026-10-02T01:57:50.438850256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0k8m5q0v6j7v3w-npdg-4-21-9gbl6-8hn7c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-svbr9q-kvqmb7zbtpsq-np-dg-svbr9q--managed/providers/Microsoft.Compute/virtualMachines/j0k8m5q0v6j7v3w-npdg-4-21-9gbl6-8hn7c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0k8m5q0v6j7v3w-npdg-4-21-9gbl6-8hn7c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-svbr9q-kvqmb7zbtpsq-np-dg-svbr9q--managed/providers/Microsoft.Compute/virtualMachines/j0k8m5q0v6j7v3w-npdg-4-21-9gbl6-8hn7c to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fef1c0\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-svbr9q\" in resourcegroup=\"rg-np-downgrade-svbr9q-kvqmb7zbtpsq\" to finish creating: GET https://rp.j3810048.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/556c3a8a-eee8-47cd-95b3-aed05e7e4d22\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/556c3a8a-eee8-47cd-95b3-aed05e7e4d22\",\n \"name\": \"556c3a8a-eee8-47cd-95b3-aed05e7e4d22\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:38:44.187977267Z\",\n \"endTime\": \"2026-10-02T01:57:50.438850256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j0k8m5q0v6j7v3w-npdg-4-21-9gbl6-8hn7c: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-svbr9q-kvqmb7zbtpsq-np-dg-svbr9q--managed/providers/Microsoft.Compute/virtualMachines/j0k8m5q0v6j7v3w-npdg-4-21-9gbl6-8hn7c to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j0k8m5q0v6j7v3w-npdg-4-21-9gbl6-8hn7c: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-svbr9q-kvqmb7zbtpsq-np-dg-svbr9q--managed/providers/Microsoft.Compute/virtualMachines/j0k8m5q0v6j7v3w-npdg-4-21-9gbl6-8hn7c to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"int","summary":{"total_runs":5,"failed_runs":0,"runs_with_occurrences":0,"runs_with_failure_pattern_matches":0,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":5,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2106093587799216128","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T18:46:55Z","started_at":"2026-10-02T18:46:55Z","completed_at":"2026-10-02T21:05:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T21:14:26.615085689Z","timing_metadata_checked_at":"2026-10-02T21:14:26.615085689Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T21:14:26.618615377Z","region_metadata_checked_at":"2026-10-02T21:14:26.618615377Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2106042797571182592","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T15:25:03Z","started_at":"2026-10-02T15:25:03Z","completed_at":"2026-10-02T17:52:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T18:02:42.595234903Z","timing_metadata_checked_at":"2026-10-02T18:02:42.595234903Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T18:02:42.598414193Z","region_metadata_checked_at":"2026-10-02T18:02:42.598414193Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105982902356414464","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T11:27:05Z","started_at":"2026-10-02T11:27:05Z","completed_at":"2026-10-02T13:50:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T14:02:04.465347334Z","timing_metadata_checked_at":"2026-10-02T14:02:04.465347334Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T14:02:04.471484713Z","region_metadata_checked_at":"2026-10-02T14:02:04.471484713Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105915723598532608","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T07:00:09Z","started_at":"2026-10-02T07:00:09Z","completed_at":"2026-10-02T09:14:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T09:29:16.459808962Z","timing_metadata_checked_at":"2026-10-02T09:29:16.459808962Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T09:29:16.462881452Z","region_metadata_checked_at":"2026-10-02T09:29:16.462881452Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105861603688714240","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T03:25:05Z","started_at":"2026-10-02T03:25:05Z","completed_at":"2026-10-02T05:52:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T06:10:48.793854777Z","timing_metadata_checked_at":"2026-10-02T06:10:48.793854777Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T06:10:48.796957067Z","region_metadata_checked_at":"2026-10-02T06:10:48.796957067Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":16,"failed_runs":9,"runs_with_occurrences":9,"runs_with_failure_pattern_matches":9,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":7,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106168378287722496","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T23:44:06Z","started_at":"2026-10-02T23:44:06Z","completed_at":"2026-10-03T02:31:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T02:39:20.600248123Z","timing_metadata_checked_at":"2026-10-03T02:39:20.600248123Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T02:39:20.60418951Z","region_metadata_checked_at":"2026-10-03T02:39:20.60418951Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106164802488373248","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T23:29:54Z","started_at":"2026-10-02T23:29:54Z","completed_at":"2026-10-03T02:10:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T02:18:30.491532763Z","timing_metadata_checked_at":"2026-10-03T02:18:30.491532763Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T02:18:30.494570754Z","region_metadata_checked_at":"2026-10-03T02:18:30.494570754Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0ef2e346d8dfb252e66e2042bfa005d0a7260c98cee62331cc2018bb267d3891","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106164802488373248","occurred_at":"2026-10-02T23:29:54Z","failed_at":"e2e","signature_id":"1697e8a0579901a891fca8403d65d3313f3b3f1b1fd0fc7f82c8b7a80c503289","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jcbqlv\" for cluster \"cluster-candidate-4-20-jcbqlv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002b94f00\u003e: \n failed to create NodePool np-jcbqlv: failed waiting for nodepool=\"np-jcbqlv\" for cluster \"cluster-candidate-4-20-jcbqlv\" in resourcegroup=\"rg-candidate-4-20-lvv7b2lssbpl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84cb018b-4347-4f56-b1c7-88d7604492f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84cb018b-4347-4f56-b1c7-88d7604492f9\",\n \"name\": \"84cb018b-4347-4f56-b1c7-88d7604492f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:32:09.437716177Z\",\n \"endTime\": \"2026-10-03T00:51:11.761764992Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c1t3y9h8f3v9w7t-np-jcbqlv-zlpbm-ww6xn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lvv7b2lssbpl-candidate-jcbqlv--managed/providers/Microsoft.Compute/virtualMachines/c1t3y9h8f3v9w7t-np-jcbqlv-zlpbm-ww6xn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c1t3y9h8f3v9w7t-np-jcbqlv-zlpbm-ww6xn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lvv7b2lssbpl-candidate-jcbqlv--managed/providers/Microsoft.Compute/virtualMachines/c1t3y9h8f3v9w7t-np-jcbqlv-zlpbm-ww6xn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jcbqlv\" for cluster \"cluster-candidate-4-20-jcbqlv\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002b94f00\u003e: \n failed to create NodePool np-jcbqlv: failed waiting for nodepool=\"np-jcbqlv\" for cluster \"cluster-candidate-4-20-jcbqlv\" in resourcegroup=\"rg-candidate-4-20-lvv7b2lssbpl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84cb018b-4347-4f56-b1c7-88d7604492f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84cb018b-4347-4f56-b1c7-88d7604492f9\",\n \"name\": \"84cb018b-4347-4f56-b1c7-88d7604492f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:32:09.437716177Z\",\n \"endTime\": \"2026-10-03T00:51:11.761764992Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c1t3y9h8f3v9w7t-np-jcbqlv-zlpbm-ww6xn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lvv7b2lssbpl-candidate-jcbqlv--managed/providers/Microsoft.Compute/virtualMachines/c1t3y9h8f3v9w7t-np-jcbqlv-zlpbm-ww6xn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c1t3y9h8f3v9w7t-np-jcbqlv-zlpbm-ww6xn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lvv7b2lssbpl-candidate-jcbqlv--managed/providers/Microsoft.Compute/virtualMachines/c1t3y9h8f3v9w7t-np-jcbqlv-zlpbm-ww6xn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"95f3d8ca95d031b58261e31a3092e237e46a1560c3640b459475dd8ad818ac15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106164802488373248","occurred_at":"2026-10-02T23:29:54Z","failed_at":"e2e","signature_id":"dab5b4064cf2f8bafae3a0f4545840436971881777855461aacd6c0158a34e84","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00113e680\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-9tnt4zgp45ww\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84e6a04c-39e2-4627-85f5-80cd1669879f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84e6a04c-39e2-4627-85f5-80cd1669879f\",\n \"name\": \"84e6a04c-39e2-4627-85f5-80cd1669879f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:58:26.609352403Z\",\n \"endTime\": \"2026-10-03T00:17:31.644821105Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-mmbgh-jld78: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-9tnt4zgp45ww--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-mmbgh-jld78 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-mmbgh-jld78: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-9tnt4zgp45ww--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-mmbgh-jld78 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00113e680\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-9tnt4zgp45ww\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84e6a04c-39e2-4627-85f5-80cd1669879f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/84e6a04c-39e2-4627-85f5-80cd1669879f\",\n \"name\": \"84e6a04c-39e2-4627-85f5-80cd1669879f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:58:26.609352403Z\",\n \"endTime\": \"2026-10-03T00:17:31.644821105Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-mmbgh-jld78: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-9tnt4zgp45ww--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-mmbgh-jld78 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-mmbgh-jld78: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-9tnt4zgp45ww--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-mmbgh-jld78 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-9tnt4zgp45ww\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106164466826612736","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T23:28:34Z","started_at":"2026-10-02T23:28:34Z","completed_at":"2026-10-03T01:53:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T02:07:34.139843814Z","timing_metadata_checked_at":"2026-10-03T02:07:34.139843814Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T02:07:34.142949403Z","region_metadata_checked_at":"2026-10-03T02:07:34.142949403Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fe651df4e0ceb2b76cd3e10304b4f6e5d26092a9a4c5b84216c92e4b5fbb7451","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106164466826612736","occurred_at":"2026-10-02T23:28:34Z","failed_at":"e2e","signature_id":"e7b07f9d924b20c7993315c8c5888cf9fdce0c07fa2995bd79dacecadf4b3449","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a814a0\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a814a0\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T23:00:43Z","started_at":"2026-10-02T23:00:43Z","completed_at":"2026-10-03T01:01:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T01:15:22.80868978Z","timing_metadata_checked_at":"2026-10-03T01:15:22.80868978Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T01:15:22.811931269Z","region_metadata_checked_at":"2026-10-03T01:15:22.811931269Z"},"failed_at":["e2e"],"failed_test_count":18,"occurrences":[{"row_id":"02789f548dcb1d7533c5a3c75b9e80c5b7b7e56a43d61e68e65d774bbf887b65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"8c026eae5a564a740588cd6182c72840e2aa3a5641704bfed2b2826360179bb7","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-kf8zjf\" to 4.21.0-0.nightly-multi-2026-10-02-021649\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000d32ae0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-9wcjt7cq5sl5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-kf8zjf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-kf8zjf\" to 4.21.0-0.nightly-multi-2026-10-02-021649\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000d32ae0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-9wcjt7cq5sl5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-kf8zjf\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2a24d6c3580df7db72e1adcf5f15873ebdc71a60b7da8ee2cc22dcad0afb737a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"a1207e94ae359351ce43ded44cf40a8ca31a697debc403c5ff1ba3520073a5d1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000cdbad0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-hbvfkpmwjx97/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000cdbad0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-hbvfkpmwjx97/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2b792e78f0b90c31557e18843f5b254a9257d2f36466aeb797d8f38cf2a3b6c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"a6fc07cc46d1de9d0be98eb74eac2c7a37dadedfaf50fe64d7a5fc997a6e6199","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-s49xkm\" for cluster \"cluster-nightly-4-20-s49xkm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad8640\u003e: \n failed to create NodePool np-s49xkm: failed starting nodepool creation \"np-s49xkm\" for cluster \"cluster-nightly-4-20-s49xkm\" in resourcegroup=\"rg-nightly-4-20-mhnxwl8ndm9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-mhnxwl8ndm9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-s49xkm/nodePools/np-s49xkm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-s49xkm\" for cluster \"cluster-nightly-4-20-s49xkm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad8640\u003e: \n failed to create NodePool np-s49xkm: failed starting nodepool creation \"np-s49xkm\" for cluster \"cluster-nightly-4-20-s49xkm\" in resourcegroup=\"rg-nightly-4-20-mhnxwl8ndm9g\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-mhnxwl8ndm9g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-s49xkm/nodePools/np-s49xkm\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2c98d446d0523f9d39a718d680644b57fe538059d304af75c42fa67eb20b4159","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"1ee54be989269d06d918f548f16f345dde8256e4f22ea3fa524967705e641986","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0023c62d0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-w5hzr8g77c9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0023c62d0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-w5hzr8g77c9w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3134cc3af095b9de5cbc9e89b47844c6f0878ec66b94870eab437cabe1a8c8e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"c08d5584a2875af5aba795c90cf9a5f747f4b83ece37210f6010dfc14be9fcbb","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012cc100\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qmt425cc9vkv\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qmt425cc9vkv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012cc100\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-qmt425cc9vkv\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-qmt425cc9vkv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3689d6050b24ee668782ea8773b0c3f3e1abf2b74ad2cce97b3f6f4b9e9d074f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"e623aa212799cd7468fc066975cf4952233ef2970cdf06136f4259200e5213d6","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006110a0\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-bh8wcncrztgb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-bh8wcncrztgb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006110a0\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-bh8wcncrztgb\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-bh8wcncrztgb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"39d2b8ec1b23d77630bf2e3c373d572a819d4806dee26c294a5d8ef202020e47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"186419d7eeb6d1b9338dd85d07548b5761dc5a9845608489d17f088d86b70742","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-pwn8nb at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000692a40\u003e: \n failed to create HCP cluster np-skip-minor-pwn8nb: failed waiting for cluster=\"np-skip-minor-pwn8nb\" in resourcegroup=\"rg-np-skip-minor-pwn8nb-fcqmqdxvjmqx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9282a0a5-6015-4ac4-8987-2148df07fa63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9282a0a5-6015-4ac4-8987-2148df07fa63\",\n \"name\": \"9282a0a5-6015-4ac4-8987-2148df07fa63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:07:14.845401374Z\",\n \"endTime\": \"2026-10-03T00:36:15.664777437Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-pwn8nb at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000692a40\u003e: \n failed to create HCP cluster np-skip-minor-pwn8nb: failed waiting for cluster=\"np-skip-minor-pwn8nb\" in resourcegroup=\"rg-np-skip-minor-pwn8nb-fcqmqdxvjmqx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9282a0a5-6015-4ac4-8987-2148df07fa63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9282a0a5-6015-4ac4-8987-2148df07fa63\",\n \"name\": \"9282a0a5-6015-4ac4-8987-2148df07fa63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:07:14.845401374Z\",\n \"endTime\": \"2026-10-03T00:36:15.664777437Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4d33c61241173b793f6e12d7a34602cbe4c6a3b0d547732f908fc5115337710d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"920ef8c829f2651cec455f2ab174b7a8fc2464390bf62513498b0be52deafbaf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-nightly-4-21-8rfqwl\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0xc00127afc0\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-nightly-4-21-8rfqwl\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0xc00127afc0\u003e: \n VerifyBasicAccess failed: failed to list services: Unauthorized\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized"}},{"row_id":"63a8499b4d7d0a6ad3783029f6dd95a1b52ed358598c18a3f3ce43f051fd3ca1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"e2515b4f86316b07a93c73fac280cf1226aaf9e7e102ff834bd8cf7ac063eeb4","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-fts22d\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111ac60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-fts22d: failed waiting for cluster=\"cp-ystream-upgrade-5-0-fts22d\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mljdp8jfhscw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ec54c6-385b-4f23-8fbf-a1f041a3140c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ec54c6-385b-4f23-8fbf-a1f041a3140c\",\n \"name\": \"51ec54c6-385b-4f23-8fbf-a1f041a3140c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:43:28.788659857Z\",\n \"endTime\": \"2026-10-03T00:12:29.488557875Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-fts22d\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00111ac60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-fts22d: failed waiting for cluster=\"cp-ystream-upgrade-5-0-fts22d\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mljdp8jfhscw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ec54c6-385b-4f23-8fbf-a1f041a3140c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/51ec54c6-385b-4f23-8fbf-a1f041a3140c\",\n \"name\": \"51ec54c6-385b-4f23-8fbf-a1f041a3140c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:43:28.788659857Z\",\n \"endTime\": \"2026-10-03T00:12:29.488557875Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7ab9bc59826f91c880e5cb96208e184a603ed88cbf996b4967a29e6122e8b529","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"4b50be6c928dad451e0a16acae2be5344c9b1cb81793cb6733f989ce8d32f9a0","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015c28d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-czq6qgmpwn7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015c28d0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-czq6qgmpwn7z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"97875fa4aea4cdface5c847ef463c883ea2025fb01749472107f5358f7fce60e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"6f3a584648b311141dc993e8d6d0a3d7960cdb4329e3b86c2f8515d7f22d3562","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-d88bcx\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010032f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-bh6j7gtlcdf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-d88bcx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-d88bcx\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010032f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-bh6j7gtlcdf2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-d88bcx\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b0af0eeeabdea08af29f9a3e62e369e9309141941557205eecf68fea42e14a6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"cd7a3a59de4b9b85fb5e749082d4ae88d69c42b7c6f7ce1a801f5b153ec8cfcb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-m2xdl7kgwht2/cluster-nightly-4-22-q779x6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c8f00\u003e: \n failed to create HCP cluster cluster-nightly-4-22-q779x6: failed waiting for cluster=\"cluster-nightly-4-22-q779x6\" in resourcegroup=\"rg-nightly-4-22-m2xdl7kgwht2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecf6db04-6bc9-42e0-9339-05a023c3c7c2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecf6db04-6bc9-42e0-9339-05a023c3c7c2\",\n \"name\": \"ecf6db04-6bc9-42e0-9339-05a023c3c7c2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:39:38.855575535Z\",\n \"endTime\": \"2026-10-03T00:08:39.476306937Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-m2xdl7kgwht2/cluster-nightly-4-22-q779x6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c8f00\u003e: \n failed to create HCP cluster cluster-nightly-4-22-q779x6: failed waiting for cluster=\"cluster-nightly-4-22-q779x6\" in resourcegroup=\"rg-nightly-4-22-m2xdl7kgwht2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecf6db04-6bc9-42e0-9339-05a023c3c7c2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ecf6db04-6bc9-42e0-9339-05a023c3c7c2\",\n \"name\": \"ecf6db04-6bc9-42e0-9339-05a023c3c7c2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:39:38.855575535Z\",\n \"endTime\": \"2026-10-03T00:08:39.476306937Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b2f572b0ae5af15ed0484f87d2a698a6077ebfa827158b69aa9bdf4ade290f44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"77b7a08a95ca74a51e809832a46efcf6d198f5ac4024bb5c02fd3092de3b8ae4","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d72ea0\u003e: \n failed to create HCP cluster np-dg-minor-x975lx: failed waiting for cluster=\"np-dg-minor-x975lx\" in resourcegroup=\"rg-np-dg-minor-x975lx-snvpx8zstwzg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1bf1cc23-284f-4b81-90c6-b740356d67dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1bf1cc23-284f-4b81-90c6-b740356d67dd\",\n \"name\": \"1bf1cc23-284f-4b81-90c6-b740356d67dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:09:01.246824643Z\",\n \"endTime\": \"2026-10-03T00:38:05.670946046Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d72ea0\u003e: \n failed to create HCP cluster np-dg-minor-x975lx: failed waiting for cluster=\"np-dg-minor-x975lx\" in resourcegroup=\"rg-np-dg-minor-x975lx-snvpx8zstwzg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1bf1cc23-284f-4b81-90c6-b740356d67dd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1bf1cc23-284f-4b81-90c6-b740356d67dd\",\n \"name\": \"1bf1cc23-284f-4b81-90c6-b740356d67dd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-03T00:09:01.246824643Z\",\n \"endTime\": \"2026-10-03T00:38:05.670946046Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bbe178568ae673b1907d59868d984f73a5fda31872ff4ad845abc192601cda2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"0f169f07ead084859a44529a3b94fb278582797703aea14168e2772c2c7db8ff","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-qssg82\" to 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014d0480\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-cz4jzkrzbs6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-qssg82\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-qssg82\" to 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014d0480\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-cz4jzkrzbs6n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-qssg82\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"be28dc2983dec672df3db3af1ea6da7d7137413c3aae0330991ff550f020b407","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"de7188ccaefa33da566dc29502073e498832d945f704d391a4cc462850f14b54","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0026fb2f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cr2tzrn9kjll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0026fb2f0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-cr2tzrn9kjll/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c0fdc9564e1937497b51ee10e32712353bf8407ef5722c433d08577332dbd3cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"d6d9357eb55b1324baafacf4251de65742fcbe5f370217da7986cfa36dfd7d18","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de8120\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-4slk4t\" in resourcegroup=\"rg-np-version-upgrade-4slk4t-r4jcn6z5vwbl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4slk4t-r4jcn6z5vwbl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-4slk4t/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000de8120\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-4slk4t\" in resourcegroup=\"rg-np-version-upgrade-4slk4t-r4jcn6z5vwbl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4slk4t-r4jcn6z5vwbl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-4slk4t/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"cb64719712209cfd480f72ffd88d49afddeb6deff2186aff041f454393d46288","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"94ac0fae6eb7b726dd0f0c2a320cf1ff6e8a675fbe183bc6c4fcd848bb4be552","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001074300\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-vhhr845dxfj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001074300\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-vhhr845dxfj7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f62f16ccdfd9e0587e8c075c50029f89670a17267e8b6325f1bff9fa9005db31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2106157461642678272","occurred_at":"2026-10-02T23:00:43Z","failed_at":"e2e","signature_id":"57529b346c23e8bbddb55077f13585dbd6f84c255cf953b120f390392faae3cd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-wlr9js\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d5cb20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-wlr9js: failed waiting for cluster=\"cp-ystream-upgrade-4-23-wlr9js\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-hsrbsrxk7qbw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9a34556-9c6b-44f5-9047-c18c0b099bd1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9a34556-9c6b-44f5-9047-c18c0b099bd1\",\n \"name\": \"c9a34556-9c6b-44f5-9047-c18c0b099bd1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:42:52.038313719Z\",\n \"endTime\": \"2026-10-03T00:11:59.486058349Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-wlr9js\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d5cb20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-wlr9js: failed waiting for cluster=\"cp-ystream-upgrade-4-23-wlr9js\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-hsrbsrxk7qbw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9a34556-9c6b-44f5-9047-c18c0b099bd1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c9a34556-9c6b-44f5-9047-c18c0b099bd1\",\n \"name\": \"c9a34556-9c6b-44f5-9047-c18c0b099bd1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:42:52.038313719Z\",\n \"endTime\": \"2026-10-03T00:11:59.486058349Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":18,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc"],"matched_occurrences":18,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106112889331912704","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T18:24:14Z","started_at":"2026-10-02T18:24:14Z","completed_at":"2026-10-02T22:39:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T22:48:12.704050398Z","timing_metadata_checked_at":"2026-10-02T22:48:12.704050398Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T22:48:12.711699071Z","region_metadata_checked_at":"2026-10-02T22:48:12.711699071Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106112516395372544","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T18:24:00Z","started_at":"2026-10-02T18:24:00Z","completed_at":"2026-10-02T22:36:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T22:48:12.577435947Z","timing_metadata_checked_at":"2026-10-02T22:48:12.577435947Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T22:48:12.581032635Z","region_metadata_checked_at":"2026-10-02T22:48:12.581032635Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106076333636325376","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T17:38:21Z","started_at":"2026-10-02T17:38:21Z","completed_at":"2026-10-02T20:22:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:31:44.008556959Z","timing_metadata_checked_at":"2026-10-02T20:31:44.008556959Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:31:44.01158195Z","region_metadata_checked_at":"2026-10-02T20:31:44.01158195Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"128c257d72627193468842a533b97d3e8f631a67ab143b8e8ed2cb1b1b3688f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106076333636325376","occurred_at":"2026-10-02T17:38:21Z","failed_at":"e2e","signature_id":"62b86d5a3e6a733894f681784985f719ee5021bcc2f645da75704b658ea44a3b","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0022ce920\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-fgvr8lf6bszk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/db510478-c50d-4f9a-86b6-3bb190ec8daa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/db510478-c50d-4f9a-86b6-3bb190ec8daa\",\n \"name\": \"db510478-c50d-4f9a-86b6-3bb190ec8daa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T17:45:05.919871003Z\",\n \"endTime\": \"2026-10-02T18:14:10.848146933Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0022ce920\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-fgvr8lf6bszk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/db510478-c50d-4f9a-86b6-3bb190ec8daa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/db510478-c50d-4f9a-86b6-3bb190ec8daa\",\n \"name\": \"db510478-c50d-4f9a-86b6-3bb190ec8daa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T17:45:05.919871003Z\",\n \"endTime\": \"2026-10-02T18:14:10.848146933Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f6f8a45f511c8e754a4eee39b9fdf833c0939ef5f57997dd2cc64c4b72fa86df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106076333636325376","occurred_at":"2026-10-02T17:38:21Z","failed_at":"e2e","signature_id":"30027dd2f2a33112ce6deb686950ad163640c21153815960521248c37fc725ac","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001361080\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001361080\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1a6b16909d875148eb73d8a8d0f0da053dc893160e2a74961d7b7fb0e70b9ee1","b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106076270763708416","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T17:38:06Z","started_at":"2026-10-02T17:38:06Z","completed_at":"2026-10-02T20:03:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:10:41.462981031Z","timing_metadata_checked_at":"2026-10-02T20:10:41.462981031Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:10:41.465897622Z","region_metadata_checked_at":"2026-10-02T20:10:41.465897622Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106076110985891840","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T17:37:28Z","started_at":"2026-10-02T17:37:28Z","completed_at":"2026-10-02T20:07:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:21:15.303422339Z","timing_metadata_checked_at":"2026-10-02T20:21:15.303422339Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:21:15.308668523Z","region_metadata_checked_at":"2026-10-02T20:21:15.308668523Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"786a6e96a1061fa64b8393de77982d022f310c80b0e4aa206a319ca4428a1a6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106076110985891840","occurred_at":"2026-10-02T17:37:28Z","failed_at":"e2e","signature_id":"78cecc53a1a8ce22aa0fd990c7f001c7f47cf04b78bf9ddf6467a4bbfd0bd07b","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc001253878\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node c7z4o0b2f4w6f5y-np-1-7fd77-f8d28: failed to get pod logs for pod fips-check-z86wg in namespace e2e-fips-test-pdk46 on node c7z4o0b2f4w6f5y-np-1-7fd77-f8d28 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-z86wg)\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc001253878\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node c7z4o0b2f4w6f5y-np-1-7fd77-f8d28: failed to get pod logs for pod fips-check-z86wg in namespace e2e-fips-test-pdk46 on node c7z4o0b2f4w6f5y-np-1-7fd77-f8d28 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-z86wg)\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ef2e8b98a5652f0f5901730ad03ac32b3b64eb01793126bb1f2548067a726e46","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-\u003cid\u003e in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node c7z4o0b2f4w6f5y-np-1-7fd77-f8d28: failed to get pod logs for pod fips-check-z86wg in namespace e2e-fips-test-pdk46 on node c7z4o0b2f4w6f5y-np-1-7fd77-f8d28 after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-z86wg)"}},{"row_id":"90ca01c83c6911cfed2f96ee9a0e6be1c5c8429e8037e36db01093888653d43c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106076110985891840","occurred_at":"2026-10-02T17:37:28Z","failed_at":"e2e","signature_id":"fd3f8d18af553b9e49373502c1a7089e0a090edeedbc36abee7cd4997c0c8596","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c93500\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-tlcz46v8f5nd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9d16ce0e-bad0-4102-85e4-8ff744b9a2fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9d16ce0e-bad0-4102-85e4-8ff744b9a2fd\",\n \"name\": \"9d16ce0e-bad0-4102-85e4-8ff744b9a2fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:19:55.216406379Z\",\n \"endTime\": \"2026-10-02T19:39:00.624366265Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-9wdrv-tkpqf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-tlcz46v8f5nd--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-9wdrv-tkpqf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-9wdrv-tkpqf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-tlcz46v8f5nd--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-9wdrv-tkpqf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c93500\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-tlcz46v8f5nd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9d16ce0e-bad0-4102-85e4-8ff744b9a2fd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9d16ce0e-bad0-4102-85e4-8ff744b9a2fd\",\n \"name\": \"9d16ce0e-bad0-4102-85e4-8ff744b9a2fd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:19:55.216406379Z\",\n \"endTime\": \"2026-10-02T19:39:00.624366265Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-9wdrv-tkpqf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-tlcz46v8f5nd--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-9wdrv-tkpqf to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-9wdrv-tkpqf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-tlcz46v8f5nd--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-9wdrv-tkpqf to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","ef2e8b98a5652f0f5901730ad03ac32b3b64eb01793126bb1f2548067a726e46"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106076013422186496","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T17:37:05Z","started_at":"2026-10-02T17:37:05Z","completed_at":"2026-10-02T20:02:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:10:41.293816556Z","timing_metadata_checked_at":"2026-10-02T20:10:41.293816556Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:10:41.296659247Z","region_metadata_checked_at":"2026-10-02T20:10:41.296659247Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106027094411579392","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T14:22:42Z","started_at":"2026-10-02T14:22:42Z","completed_at":"2026-10-02T17:15:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:30:48.352536557Z","timing_metadata_checked_at":"2026-10-02T17:30:48.352536557Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T17:30:48.355730946Z","region_metadata_checked_at":"2026-10-02T17:30:48.355730946Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"50c933e6d5898b385510e4d5e3600f5ee50e4978398af365eba1bae197572cdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106027094411579392","occurred_at":"2026-10-02T14:22:42Z","failed_at":"e2e","signature_id":"027fa5f48a6be3eecdd7f20a836febd0f6b94c7081ce6eec02e3c40541cd6485","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109c260\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-vl5bmxfjpvnl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/76a0d1d5-c260-480b-9486-2626418cd0aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/76a0d1d5-c260-480b-9486-2626418cd0aa\",\n \"name\": \"76a0d1d5-c260-480b-9486-2626418cd0aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:16:07.904782817Z\",\n \"endTime\": \"2026-10-02T15:35:14.636915839Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p5k5p2r5e7o0k9d-np-1-pmf67-7nch9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-vl5bmxfjpvnl--managed/providers/Microsoft.Compute/virtualMachines/p5k5p2r5e7o0k9d-np-1-pmf67-7nch9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p5k5p2r5e7o0k9d-np-1-pmf67-7nch9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-vl5bmxfjpvnl--managed/providers/Microsoft.Compute/virtualMachines/p5k5p2r5e7o0k9d-np-1-pmf67-7nch9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00109c260\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-vl5bmxfjpvnl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/76a0d1d5-c260-480b-9486-2626418cd0aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/76a0d1d5-c260-480b-9486-2626418cd0aa\",\n \"name\": \"76a0d1d5-c260-480b-9486-2626418cd0aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:16:07.904782817Z\",\n \"endTime\": \"2026-10-02T15:35:14.636915839Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p5k5p2r5e7o0k9d-np-1-pmf67-7nch9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-vl5bmxfjpvnl--managed/providers/Microsoft.Compute/virtualMachines/p5k5p2r5e7o0k9d-np-1-pmf67-7nch9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p5k5p2r5e7o0k9d-np-1-pmf67-7nch9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-vl5bmxfjpvnl--managed/providers/Microsoft.Compute/virtualMachines/p5k5p2r5e7o0k9d-np-1-pmf67-7nch9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b4fb57210cb77dd893d259acc6360fa6b8ad44b98ff3eadc92728102dc68cc8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106027094411579392","occurred_at":"2026-10-02T14:22:42Z","failed_at":"e2e","signature_id":"b41c51e9e9b5408d17767da6eb46133657c442a5608f7e5cadaee9c0bae91a0f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qx4psk\" for cluster \"cluster-candidate-5-1-qx4psk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013d5b40\u003e: \n failed to create NodePool np-qx4psk: failed waiting for nodepool=\"np-qx4psk\" for cluster \"cluster-candidate-5-1-qx4psk\" in resourcegroup=\"rg-candidate-5-1-kt5m7922qfx8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/08a3ce96-4944-47f9-8702-e2ad9f78a6cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/08a3ce96-4944-47f9-8702-e2ad9f78a6cf\",\n \"name\": \"08a3ce96-4944-47f9-8702-e2ad9f78a6cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:30:38.36539796Z\",\n \"endTime\": \"2026-10-02T15:49:40.880753059Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z1v0n8y8q7v5r0b-np-qx4psk-v8zlj-7bpw8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kt5m7922qfx8-candidate-qx4psk--managed/providers/Microsoft.Compute/virtualMachines/z1v0n8y8q7v5r0b-np-qx4psk-v8zlj-7bpw8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z1v0n8y8q7v5r0b-np-qx4psk-v8zlj-7bpw8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kt5m7922qfx8-candidate-qx4psk--managed/providers/Microsoft.Compute/virtualMachines/z1v0n8y8q7v5r0b-np-qx4psk-v8zlj-7bpw8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qx4psk\" for cluster \"cluster-candidate-5-1-qx4psk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013d5b40\u003e: \n failed to create NodePool np-qx4psk: failed waiting for nodepool=\"np-qx4psk\" for cluster \"cluster-candidate-5-1-qx4psk\" in resourcegroup=\"rg-candidate-5-1-kt5m7922qfx8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/08a3ce96-4944-47f9-8702-e2ad9f78a6cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/08a3ce96-4944-47f9-8702-e2ad9f78a6cf\",\n \"name\": \"08a3ce96-4944-47f9-8702-e2ad9f78a6cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T15:30:38.36539796Z\",\n \"endTime\": \"2026-10-02T15:49:40.880753059Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z1v0n8y8q7v5r0b-np-qx4psk-v8zlj-7bpw8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kt5m7922qfx8-candidate-qx4psk--managed/providers/Microsoft.Compute/virtualMachines/z1v0n8y8q7v5r0b-np-qx4psk-v8zlj-7bpw8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z1v0n8y8q7v5r0b-np-qx4psk-v8zlj-7bpw8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kt5m7922qfx8-candidate-qx4psk--managed/providers/Microsoft.Compute/virtualMachines/z1v0n8y8q7v5r0b-np-qx4psk-v8zlj-7bpw8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106027031710928896","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T14:22:22Z","started_at":"2026-10-02T14:22:22Z","completed_at":"2026-10-02T17:09:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:20:04.064055694Z","timing_metadata_checked_at":"2026-10-02T17:20:04.064055694Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T17:20:04.067048585Z","region_metadata_checked_at":"2026-10-02T17:20:04.067048585Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106026848772165632","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-02T14:21:43Z","started_at":"2026-10-02T14:21:43Z","completed_at":"2026-10-02T17:03:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:20:03.783204653Z","timing_metadata_checked_at":"2026-10-02T17:20:03.783204653Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T17:20:03.78732784Z","region_metadata_checked_at":"2026-10-02T17:20:03.78732784Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106019544450469888","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T13:52:41Z","started_at":"2026-10-02T13:52:41Z","completed_at":"2026-10-02T16:54:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T17:09:56.176158361Z","timing_metadata_checked_at":"2026-10-02T17:09:56.176158361Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T17:09:56.17972525Z","region_metadata_checked_at":"2026-10-02T17:09:56.17972525Z"},"failed_at":["e2e"],"failed_test_count":7,"occurrences":[{"row_id":"2fbf05d5b887de006773213b9bb548397e43cbfb2fd220732c9be0d105353a5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106019544450469888","occurred_at":"2026-10-02T13:52:41Z","failed_at":"e2e","signature_id":"de523b67e0a6d9a60445a4ba52912609f2789ba54d2cc4d853ae5181e2dacd65","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-494d7tjl9z6l/cluster-candidate-4-20-zdwtn5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c31000\u003e: \n failed to create HCP cluster cluster-candidate-4-20-zdwtn5: failed waiting for cluster=\"cluster-candidate-4-20-zdwtn5\" in resourcegroup=\"rg-candidate-4-20-494d7tjl9z6l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/cecb851f-dab3-443d-be30-9e210495bb30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/cecb851f-dab3-443d-be30-9e210495bb30\",\n \"name\": \"cecb851f-dab3-443d-be30-9e210495bb30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:47:02.598518687Z\",\n \"endTime\": \"2026-10-02T15:16:11.481682438Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-494d7tjl9z6l/cluster-candidate-4-20-zdwtn5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c31000\u003e: \n failed to create HCP cluster cluster-candidate-4-20-zdwtn5: failed waiting for cluster=\"cluster-candidate-4-20-zdwtn5\" in resourcegroup=\"rg-candidate-4-20-494d7tjl9z6l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/cecb851f-dab3-443d-be30-9e210495bb30\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/cecb851f-dab3-443d-be30-9e210495bb30\",\n \"name\": \"cecb851f-dab3-443d-be30-9e210495bb30\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:47:02.598518687Z\",\n \"endTime\": \"2026-10-02T15:16:11.481682438Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"175d989daf815f0cdb482dfd688da785ab35fcb3d5ce206297ee4441148c7f34","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3689683abaa534cfd061894c4597b724deee45fbffaeaab24bf3a45de8785b35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106019544450469888","occurred_at":"2026-10-02T13:52:41Z","failed_at":"e2e","signature_id":"05c5c60df4886da0c5e4f9f568cb4f19d6e36d935d4f9287ce0318771bf33cd2","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002486980\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-4z7j926fljlc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f9754b3c-7f81-4d15-8726-14659a0f0d2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f9754b3c-7f81-4d15-8726-14659a0f0d2b\",\n \"name\": \"f9754b3c-7f81-4d15-8726-14659a0f0d2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:15:50.839633613Z\",\n \"endTime\": \"2026-10-02T14:44:52.180223552Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002486980\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-4z7j926fljlc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f9754b3c-7f81-4d15-8726-14659a0f0d2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f9754b3c-7f81-4d15-8726-14659a0f0d2b\",\n \"name\": \"f9754b3c-7f81-4d15-8726-14659a0f0d2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:15:50.839633613Z\",\n \"endTime\": \"2026-10-02T14:44:52.180223552Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"25d62dff0f4ef1dbd97529af39bb1aa49db23fb054d6258af57d42dbe4d2e175","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"46c8dcc2e02d525311ffcda3e35f094a82d990b785fb63b409e3637af438c102","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106019544450469888","occurred_at":"2026-10-02T13:52:41Z","failed_at":"e2e","signature_id":"772938ecd08ea1b121d1c5bf4bcf99e8fc544e656eb4dd2e02953603c6e51b73","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-48fc7bfssxlp/cluster-candidate-4-22-7klbxb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d40740\u003e: \n failed to create HCP cluster cluster-candidate-4-22-7klbxb: failed waiting for cluster=\"cluster-candidate-4-22-7klbxb\" in resourcegroup=\"rg-candidate-4-22-48fc7bfssxlp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3f2efc0e-134f-4975-8954-ac850a668709\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3f2efc0e-134f-4975-8954-ac850a668709\",\n \"name\": \"3f2efc0e-134f-4975-8954-ac850a668709\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:48:41.860181794Z\",\n \"endTime\": \"2026-10-02T15:17:41.529062952Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-48fc7bfssxlp/cluster-candidate-4-22-7klbxb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d40740\u003e: \n failed to create HCP cluster cluster-candidate-4-22-7klbxb: failed waiting for cluster=\"cluster-candidate-4-22-7klbxb\" in resourcegroup=\"rg-candidate-4-22-48fc7bfssxlp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3f2efc0e-134f-4975-8954-ac850a668709\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3f2efc0e-134f-4975-8954-ac850a668709\",\n \"name\": \"3f2efc0e-134f-4975-8954-ac850a668709\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:48:41.860181794Z\",\n \"endTime\": \"2026-10-02T15:17:41.529062952Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m55s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1ef6fb2cacfb3e636cda1c995d6e001cf828dbd8df395155de323dd4dfc5ec1d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"854a2b1d12b66a3b9e46f2118bdd20c602cfafcb8c37a1da18fda005063fa9d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106019544450469888","occurred_at":"2026-10-02T13:52:41Z","failed_at":"e2e","signature_id":"7f6dbd8ea35d4d5c465b0443a395fb4e4c88a0b151df84caa10f8cb330f7504a","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ae5a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-k4q5zpfgpr4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3c73c7ae-1999-4cb4-b92d-cbffe4356bef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3c73c7ae-1999-4cb4-b92d-cbffe4356bef\",\n \"name\": \"3c73c7ae-1999-4cb4-b92d-cbffe4356bef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:14:49.716994175Z\",\n \"endTime\": \"2026-10-02T14:43:52.152352909Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ae5a0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-k4q5zpfgpr4k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3c73c7ae-1999-4cb4-b92d-cbffe4356bef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3c73c7ae-1999-4cb4-b92d-cbffe4356bef\",\n \"name\": \"3c73c7ae-1999-4cb4-b92d-cbffe4356bef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:14:49.716994175Z\",\n \"endTime\": \"2026-10-02T14:43:52.152352909Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d7783a52288ee0da7c2bb9244a8f44d7cd077d00612f443286315616360cf2f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"86c765321e435d7db79ee15632d66a63bc029c21b404765e8c8b8c80283b7153","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106019544450469888","occurred_at":"2026-10-02T13:52:41Z","failed_at":"e2e","signature_id":"c4a8525ea8ab9c69433edf6f795e882195e977264a70f0c1c09670695bafd2b9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-rf66b2pjnzs5/cluster-candidate-4-21-zp7tkm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9f080\u003e: \n failed to create HCP cluster cluster-candidate-4-21-zp7tkm: failed waiting for cluster=\"cluster-candidate-4-21-zp7tkm\" in resourcegroup=\"rg-candidate-4-21-rf66b2pjnzs5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f866fdfa-caa6-4c46-a375-3e94e0748ca1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f866fdfa-caa6-4c46-a375-3e94e0748ca1\",\n \"name\": \"f866fdfa-caa6-4c46-a375-3e94e0748ca1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:48:13.433342194Z\",\n \"endTime\": \"2026-10-02T15:17:21.452859562Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-rf66b2pjnzs5/cluster-candidate-4-21-zp7tkm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9f080\u003e: \n failed to create HCP cluster cluster-candidate-4-21-zp7tkm: failed waiting for cluster=\"cluster-candidate-4-21-zp7tkm\" in resourcegroup=\"rg-candidate-4-21-rf66b2pjnzs5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f866fdfa-caa6-4c46-a375-3e94e0748ca1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/f866fdfa-caa6-4c46-a375-3e94e0748ca1\",\n \"name\": \"f866fdfa-caa6-4c46-a375-3e94e0748ca1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:48:13.433342194Z\",\n \"endTime\": \"2026-10-02T15:17:21.452859562Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"175d989daf815f0cdb482dfd688da785ab35fcb3d5ce206297ee4441148c7f34","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c619c2409df7f9151b00157eaeae4d103c92bc1c1268344cfa2ba46ced435135","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106019544450469888","occurred_at":"2026-10-02T13:52:41Z","failed_at":"e2e","signature_id":"11810d5afd2466b985708edb6b227d43147568476512de5e557b45dc12da72bb","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002f205e0\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002f205e0\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred"}},{"row_id":"e7fe2578686bbb42223e0b1240e7d21d21a4e3556f969d37de1e8004915eb883","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2106019544450469888","occurred_at":"2026-10-02T13:52:41Z","failed_at":"e2e","signature_id":"2de512135ec095a8fb4a5a8eaa1c282688ca68221163c9737371e28a40932e93","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105d400\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-tqrndt5v6zjd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/8281d8b1-ac73-4bd5-b7ff-5e1ee0313132\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/8281d8b1-ac73-4bd5-b7ff-5e1ee0313132\",\n \"name\": \"8281d8b1-ac73-4bd5-b7ff-5e1ee0313132\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:42:30.687332757Z\",\n \"endTime\": \"2026-10-02T15:01:32.464297541Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x0e7s2g7c7i0z9g-np-1-cpzvz-fbtjm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tqrndt5v6zjd--managed/providers/Microsoft.Compute/virtualMachines/x0e7s2g7c7i0z9g-np-1-cpzvz-fbtjm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x0e7s2g7c7i0z9g-np-1-cpzvz-fbtjm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tqrndt5v6zjd--managed/providers/Microsoft.Compute/virtualMachines/x0e7s2g7c7i0z9g-np-1-cpzvz-fbtjm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105d400\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-tqrndt5v6zjd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/8281d8b1-ac73-4bd5-b7ff-5e1ee0313132\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/8281d8b1-ac73-4bd5-b7ff-5e1ee0313132\",\n \"name\": \"8281d8b1-ac73-4bd5-b7ff-5e1ee0313132\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T14:42:30.687332757Z\",\n \"endTime\": \"2026-10-02T15:01:32.464297541Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x0e7s2g7c7i0z9g-np-1-cpzvz-fbtjm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tqrndt5v6zjd--managed/providers/Microsoft.Compute/virtualMachines/x0e7s2g7c7i0z9g-np-1-cpzvz-fbtjm to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x0e7s2g7c7i0z9g-np-1-cpzvz-fbtjm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-tqrndt5v6zjd--managed/providers/Microsoft.Compute/virtualMachines/x0e7s2g7c7i0z9g-np-1-cpzvz-fbtjm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["175d989daf815f0cdb482dfd688da785ab35fcb3d5ce206297ee4441148c7f34","1ef6fb2cacfb3e636cda1c995d6e001cf828dbd8df395155de323dd4dfc5ec1d","25d62dff0f4ef1dbd97529af39bb1aa49db23fb054d6258af57d42dbe4d2e175","7d7783a52288ee0da7c2bb9244a8f44d7cd077d00612f443286315616360cf2f","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105887427791949824","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T05:07:42Z","started_at":"2026-10-02T05:07:42Z","completed_at":"2026-10-02T08:03:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T08:16:45.271852976Z","timing_metadata_checked_at":"2026-10-02T08:16:45.271852976Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T08:16:45.274980065Z","region_metadata_checked_at":"2026-10-02T08:16:45.274980065Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ec707a7106d0074285cb49fd8e3905fd2faabe9acc93dccd2229899ab9d89e50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105887427791949824","occurred_at":"2026-10-02T05:07:42Z","failed_at":"e2e","signature_id":"6b726d91a1c39219d8c566b13e4217f46505863b4bad0cac50b14614bbcfef61","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006eb260\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006eb260\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105840312533389312","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T02:00:29Z","started_at":"2026-10-02T02:00:29Z","completed_at":"2026-10-02T04:19:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T04:26:07.339991927Z","timing_metadata_checked_at":"2026-10-02T04:26:07.339991927Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T04:26:07.343759015Z","region_metadata_checked_at":"2026-10-02T04:26:07.343759015Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"2960d2604579f555bc379d9be790de4eb2b397fc93b3cc2d2c4c80c5469476f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105840312533389312","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"0f2c19322fdc13548351d0dd87b9264a36ac0224fdbdb6e5d8c9d4d0f2d1b7d2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-fnp94g\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017b8120\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-pqm748zt4v25/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnp94g\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-fnp94g\" on minor version 4.22\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017b8120\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-pqm748zt4v25/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-fnp94g\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7890d4709cf48f7f4ebef579f76672865f0ba03f04ceefa2c6e2b681c78dcbe7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105840312533389312","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"4478ddf169933e7a30cfdff318b36592603a0cde5ed5ed1b4b08976391f6d6e7","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-phqn4q\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014ae240\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-krv2vhb295c2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-phqn4q\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-phqn4q\" on minor version 4.21\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014ae240\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-krv2vhb295c2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-phqn4q\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9833be3e038de652d124f7d98f74c62fd7c977fc5cb8bb8b43348980a595f529","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105840312533389312","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"55dea6d69c44d399ab2871b15b3c61cc17770dc6cc90e71d08decff6339b02dd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-zrd7b7\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010807b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-pn6t6n4tjrff/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-zrd7b7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-zrd7b7\" on minor version 5.0\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010807b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-pn6t6n4tjrff/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-zrd7b7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c97f1db88a54279475a8cf8a3cda1153cd80a8047e401865f9532b21017a05ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105840312533389312","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"aafca1b6c4226501177109f9662a7607e5acde0d9020853bb9a39b54d56a22fb","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-9p29lj\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00153a060\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-l7kx94q698fp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9p29lj\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-9p29lj\" on minor version 4.20\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00153a060\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-l7kx94q698fp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-9p29lj\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"fb20952020f42620f2b5c7c41e9c58cbfa32024c1e7470804741a657f4184f69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105840312533389312","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"5b45b9c93b4a00d15b1924525277b33e26becad338d153090bc838b984a25609","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-xmqw4g\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00155a450\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-v87rt9mg748g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-xmqw4g\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-xmqw4g\" on minor version 4.23\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00155a450\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-v87rt9mg748g/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-xmqw4g\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"stg","summary":{"total_runs":6,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T23:00:44Z","started_at":"2026-10-02T23:00:44Z","completed_at":"2026-10-03T00:41:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-03T00:54:33.521888808Z","timing_metadata_checked_at":"2026-10-03T00:54:33.521888808Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-03T00:54:33.525661696Z","region_metadata_checked_at":"2026-10-03T00:54:33.525661696Z"},"failed_at":["e2e"],"failed_test_count":25,"occurrences":[{"row_id":"22a2fa267a120bfb0b960e647521d14d95bfa582d7976eecac20cef9ab1c69b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"a70f09283f83907ef445a20f2e96d80236d23d53ed875421b7b5e1da8a6a4b87","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-27sjww at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d81140\u003e: \n failed to create HCP cluster np-skip-minor-27sjww: failed waiting for cluster=\"np-skip-minor-27sjww\" in resourcegroup=\"rg-np-skip-minor-27sjww-kgw6mbppvdmz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n \"name\": \"74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:50:41.341932888Z\",\n \"endTime\": \"2026-10-03T00:19:42.833586106Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-27sjww at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d81140\u003e: \n failed to create HCP cluster np-skip-minor-27sjww: failed waiting for cluster=\"np-skip-minor-27sjww\" in resourcegroup=\"rg-np-skip-minor-27sjww-kgw6mbppvdmz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n \"name\": \"74064ef6-783f-4e23-b96a-a2576b7c7d5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:50:41.341932888Z\",\n \"endTime\": \"2026-10-03T00:19:42.833586106Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"23fcefc5b11d65413ea804a8089bb9a1398ada3fb4ca364014a85387b12f549b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"f944baaa2c2ba95ce8132743ac5ce22cf1d0b2f92722f2569ba0b7a850637a81","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-8cm72gr5ql2s/cluster-nightly-4-22-65xxws should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00164c880\u003e: \n failed to create HCP cluster cluster-nightly-4-22-65xxws: failed waiting for cluster=\"cluster-nightly-4-22-65xxws\" in resourcegroup=\"rg-nightly-4-22-8cm72gr5ql2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\",\n \"name\": \"562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:28:43.258428047Z\",\n \"endTime\": \"2026-10-02T23:57:43.738727793Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-8cm72gr5ql2s/cluster-nightly-4-22-65xxws should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00164c880\u003e: \n failed to create HCP cluster cluster-nightly-4-22-65xxws: failed waiting for cluster=\"cluster-nightly-4-22-65xxws\" in resourcegroup=\"rg-nightly-4-22-8cm72gr5ql2s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\",\n \"name\": \"562de0b5-c401-4c2c-b9be-9fd3eaaa9a92\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:28:43.258428047Z\",\n \"endTime\": \"2026-10-02T23:57:43.738727793Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3877ae37640449584ed6f000dd6c4c3d6187efb87205f4490a43d2f9d61d1cf0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"8579cebca7275d6feeb1cb980486c433a32dc36619b5f2acd7985e3b37d192cf","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00106c720\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-88jnzt6h65n4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00106c720\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-88jnzt6h65n4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3e7429b3223a1d6d355e131e14bce8c79647455a8b75c865c3babe5f99cb7d3d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"ec37fd56a0307f63cfac0b7076ce6db1a98af0f908355fdb720d73618014817c","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-766n78\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00149e390\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb9p7vsl4pjf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-766n78\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-766n78\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00149e390\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-qb9p7vsl4pjf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-766n78\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"41e0d49cf66aee5b2b9407196395cb8b5ed0b711efd641841a05a0ae84f7c1ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"3c45a70cfc1f78d78efba119b5ce6faa0b80d4c42bcb7dd0ee59de43bc516ac9","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00182a270\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7bxvbnqqpfz2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00182a270\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7bxvbnqqpfz2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4c1ab9d3c027b28656115e6dfb704765226717d072023348ee2328238134c91c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"1644bbcb2e6380ac66cea640064367161834cb558a92eb398848bad14721b31d","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p76rlg\" to 4.21.0-0.nightly-multi-2026-10-02-021649\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ab88a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-c4g7bpjwkcrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p76rlg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p76rlg\" to 4.21.0-0.nightly-multi-2026-10-02-021649\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ab88a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-c4g7bpjwkcrc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p76rlg\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5d55c2aebec03b92ef9b0a8ea1bbdf42b86b5bc81071271009dd8de1b9f2424b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"cd2c3947df5446ae84ba6b5d7e021afccf38c12db58a8bee834290fd0af21ba2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tqrvbk\" for cluster \"cluster-nightly-5-0-tqrvbk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f91320\u003e: \n failed to create NodePool np-tqrvbk: failed waiting for nodepool=\"np-tqrvbk\" for cluster \"cluster-nightly-5-0-tqrvbk\" in resourcegroup=\"rg-nightly-5-0-r6nmqb54mssw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/096e1e24-3a65-44de-9802-0a6339605f8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/096e1e24-3a65-44de-9802-0a6339605f8f\",\n \"name\": \"096e1e24-3a65-44de-9802-0a6339605f8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:41:44.73235146Z\",\n \"endTime\": \"2026-10-03T00:00:53.838500072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-89fp6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-d94nh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-89fp6: Creating: virtualmachine creating or updating\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-d94nh: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tqrvbk\" for cluster \"cluster-nightly-5-0-tqrvbk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f91320\u003e: \n failed to create NodePool np-tqrvbk: failed waiting for nodepool=\"np-tqrvbk\" for cluster \"cluster-nightly-5-0-tqrvbk\" in resourcegroup=\"rg-nightly-5-0-r6nmqb54mssw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/096e1e24-3a65-44de-9802-0a6339605f8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/096e1e24-3a65-44de-9802-0a6339605f8f\",\n \"name\": \"096e1e24-3a65-44de-9802-0a6339605f8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:41:44.73235146Z\",\n \"endTime\": \"2026-10-03T00:00:53.838500072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-89fp6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-d94nh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-89fp6: Creating: virtualmachine creating or updating\\nMachine s0b5t8z5y1y3a5r-np-tqrvbk-zdnfc-d94nh: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"673cf86d98764226c7affd62e1847aa6911ede20e096bbfca7b0205b94acca77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"7eff813a5363b798d17a20887832fb9ba98d961ef78f14f728e98b256371020a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-lg5p6f\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56dc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-lg5p6f: failed waiting for cluster=\"cp-ystream-upgrade-5-0-lg5p6f\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-b2ldx28lrgzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\",\n \"name\": \"9c7f2efd-8245-4407-9270-f3937d54358b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:32:39.04806912Z\",\n \"endTime\": \"2026-10-03T00:01:43.756113041Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-lg5p6f\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a56dc0\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-lg5p6f: failed waiting for cluster=\"cp-ystream-upgrade-5-0-lg5p6f\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-b2ldx28lrgzd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9c7f2efd-8245-4407-9270-f3937d54358b\",\n \"name\": \"9c7f2efd-8245-4407-9270-f3937d54358b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:32:39.04806912Z\",\n \"endTime\": \"2026-10-03T00:01:43.756113041Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6a329dc7cda791050392c86be35e3a0986db401223546d6c837c6d0d9319b070","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"770ee469d83ba1d5d599e53e42dfd2f872e81b78bafe998012dfc850626289cc","test_name":"Customer should be able to update node pool labels and taints","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00141a900\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-kmfrpjr862hw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kmfrpjr862hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00141a900\u003e: \n failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-kmfrpjr862hw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-kmfrpjr862hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02985e80f9e6dd23af129635b4bd016a473a6a57b067e6f88bbc87e7f20a5595","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6afe1a99b1b8ac172632330a1e087e0d9d8a9abc157e6ffcdf220b2847251fe5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"860eb571af757ccc8cd08de2262a7ce14edf5abeb53618b97b6ce441d905813c","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010647a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-b4pbcj49h2gk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\",\n \"name\": \"56464cc9-6bea-4036-8793-830b73f9c445\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:18:22.317106803Z\",\n \"endTime\": \"2026-10-02T23:37:23.768654646Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: Creating: virtualmachine creating or updating\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010647a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-b4pbcj49h2gk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/56464cc9-6bea-4036-8793-830b73f9c445\",\n \"name\": \"56464cc9-6bea-4036-8793-830b73f9c445\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:18:22.317106803Z\",\n \"endTime\": \"2026-10-02T23:37:23.768654646Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-882wt: Creating: virtualmachine creating or updating\\nMachine z5o3s0b7q4a6b4y-np-1-zmwpp-t6wms: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6c02b4373162132fa81b41f63efab1c64e7c948ac700e39df02be7de2a79385a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"c7e8f4fd3f2bc41bffbc9f2d6790ff6e60abc16e8cbff179fedf0313fca2092e","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-85ljr8\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001476720\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-85ljr8: failed waiting for cluster=\"cp-ystream-upgrade-4-23-85ljr8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xxm22554wqqw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4209a33e-d8fd-4e71-a9e2-6938fd555838\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4209a33e-d8fd-4e71-a9e2-6938fd555838\",\n \"name\": \"4209a33e-d8fd-4e71-a9e2-6938fd555838\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:32:20.614033798Z\",\n \"endTime\": \"2026-10-03T00:01:23.755103284Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-85ljr8\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001476720\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-85ljr8: failed waiting for cluster=\"cp-ystream-upgrade-4-23-85ljr8\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-xxm22554wqqw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4209a33e-d8fd-4e71-a9e2-6938fd555838\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4209a33e-d8fd-4e71-a9e2-6938fd555838\",\n \"name\": \"4209a33e-d8fd-4e71-a9e2-6938fd555838\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:32:20.614033798Z\",\n \"endTime\": \"2026-10-03T00:01:23.755103284Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"709d2057086c3f36e9914797fbf18ae41d1304e287bcc9470cc1a481ec31da7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"68ded3176e1d7fbcd1f8d33ace4e652b4594754aa12dcb2811b6d01aec17dc6b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00100c018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dbjg2z24dvcx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n \"name\": \"2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:17:46.069336916Z\",\n \"endTime\": \"2026-10-02T23:36:53.678900847Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-p829p-t26hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-p829p-t26hm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00100c018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-dbjg2z24dvcx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n \"name\": \"2f88e2f7-11dc-4604-9894-6c4ed35e59a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:17:46.069336916Z\",\n \"endTime\": \"2026-10-02T23:36:53.678900847Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-p829p-t26hm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-p829p-2b2s6: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-p829p-t26hm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9580141697adb7a525d718295253254da6da1a71b022f3fab8ab2e393b4bec4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"8a7d51d9f85003ad4001a7f59ed5782eb2de4e62df81b0e16178abf5c822b1aa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c80140\u003e: \n failed to create NodePool np-fpzhbk: failed waiting for nodepool=\"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\" in resourcegroup=\"rg-nightly-4-21-lf2wqrcgx8mk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n \"name\": \"f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:36:50.854597667Z\",\n \"endTime\": \"2026-10-02T23:55:53.768834785Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: Creating: virtualmachine creating or updating\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c80140\u003e: \n failed to create NodePool np-fpzhbk: failed waiting for nodepool=\"np-fpzhbk\" for cluster \"cluster-nightly-4-21-fpzhbk\" in resourcegroup=\"rg-nightly-4-21-lf2wqrcgx8mk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n \"name\": \"f07e721e-1ddd-4cdc-848f-f939d97b8b6a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:36:50.854597667Z\",\n \"endTime\": \"2026-10-02T23:55:53.768834785Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-g4dx8: Creating: virtualmachine creating or updating\\nMachine v2x8c0w5w4x9u4j-np-fpzhbk-pj8tx-z5xht: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9d8ed537bdd276ef98cae12d894161ad6ebf274e3b8aec59fc813e10234a3bcb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"a22a4ddd3b8ff67faf5b163bf25b83cdba837b1c018fd882d7a9b40a6d220ea5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gfwqg6\" to 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018044b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-p7jzbr5ffl22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gfwqg6\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-gfwqg6\" to 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0018044b0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-p7jzbr5ffl22/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-gfwqg6\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9e45ccb451c0609649fd3d064efdf69cadd76afe7e3ea1d4c3c423b9b3a4c195","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"088d80b26ce242a6a23ce42eb36d87da5a2ba3b0cb09615c4e35d5a5873f05c2","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001214a20\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r44ftv\" in resourcegroup=\"rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r44ftv/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001214a20\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r44ftv\" in resourcegroup=\"rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r44ftv-dq6wfvlkhtqj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r44ftv/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a19bac4603b0b3d3aaa37dd3da9de7f22d558ee988b8b04f53e15bf86f5f4b26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"39ae3bb12ff8df998e369fcb228b237442fff1c7483c6d65c12668d7a03441bf","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0005e2c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9rcfwwnss4kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0005e2c30\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-9rcfwwnss4kp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a228a4819a47995e2dbb8d3d9ed55d86d6815032cdbce467c80f6d441a7efb1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"a6951ac14ce4de69f00809a996ba16b47c97b931a257020baf028934f05e2b52","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002910ff0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-rbgmrr9dbzzr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002910ff0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-rbgmrr9dbzzr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a9b92d48ea08eacd35c648a32cc39c2e9371ceb1e64080e239907db8bbf94b92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"5d19c65c5cc2f83977dfc0bb6da78485985e9a73a9516d73fcb59a9e71c1d6cf","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc002b293e0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rx57wpbdxhdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\",\n \"name\": \"811670f5-5dbe-439c-9697-edae7243b031\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:47:39.993818098Z\",\n \"endTime\": \"2026-10-03T00:06:42.823230365Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: Creating: virtualmachine creating or updating\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc002b293e0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rx57wpbdxhdh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/811670f5-5dbe-439c-9697-edae7243b031\",\n \"name\": \"811670f5-5dbe-439c-9697-edae7243b031\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:47:39.993818098Z\",\n \"endTime\": \"2026-10-03T00:06:42.823230365Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-lgvpd: Creating: virtualmachine creating or updating\\nMachine e9p1n9f9j7n4p2s-np-delete-main-ntkpd-thccv: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"bf7e43a787703e9d0e519cc491df6eff4feb9ca17052d660fe3071476d9a6512","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"428e779e55c222fac1f48387e160b1242c8a6e675443dd3caec7bdc747ed9c1c","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aae3e0\u003e: \n failed to create HCP cluster np-dg-minor-t84576: failed waiting for cluster=\"np-dg-minor-t84576\" in resourcegroup=\"rg-np-dg-minor-t84576-xhl45nnpkcxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n \"name\": \"86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:50:46.033712456Z\",\n \"endTime\": \"2026-10-03T00:19:52.83457796Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aae3e0\u003e: \n failed to create HCP cluster np-dg-minor-t84576: failed waiting for cluster=\"np-dg-minor-t84576\" in resourcegroup=\"rg-np-dg-minor-t84576-xhl45nnpkcxs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n \"name\": \"86a92082-52bb-4ade-8c3c-f2f5320c6001\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:50:46.033712456Z\",\n \"endTime\": \"2026-10-03T00:19:52.83457796Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c1be8e717ca7e33e25aef11efb753f1e824cfcb0000c4b3412976f6a05202151","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"4c7eda445e675105debe286cdd5590eda5e7186b2f1edc9eaf4ca72af1eec7fa","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089caa0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b7c2z62bpd6r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\",\n \"name\": \"0c057ce9-73bf-4556-855a-010dcaa301e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:15:33.73277988Z\",\n \"endTime\": \"2026-10-02T23:34:33.89328902Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-lcxp7-kcfwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-lcxp7-kcfwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089caa0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-b7c2z62bpd6r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0c057ce9-73bf-4556-855a-010dcaa301e1\",\n \"name\": \"0c057ce9-73bf-4556-855a-010dcaa301e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:15:33.73277988Z\",\n \"endTime\": \"2026-10-02T23:34:33.89328902Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-lcxp7-kcfwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-lcxp7-kcfwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-b7c2z62bpd6r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-lcxp7-kcfwr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c9b61244728ec7adac538fde06974d1434af3a50004aa305ffe69f90c0545f0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"8eb7f7f722acb8f7d85fccf2574849a9b3348847ef0469070b23978413484f9f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vw9ss6\" for cluster \"cluster-nightly-4-23-vw9ss6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0f2c0\u003e: \n failed to create NodePool np-vw9ss6: failed waiting for nodepool=\"np-vw9ss6\" for cluster \"cluster-nightly-4-23-vw9ss6\" in resourcegroup=\"rg-nightly-4-23-66zfk978pxch\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\",\n \"name\": \"04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:42:21.716380971Z\",\n \"endTime\": \"2026-10-03T00:01:23.832580289Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-qhsr8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-wcqs9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-qhsr8: Creating: virtualmachine creating or updating\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-wcqs9: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vw9ss6\" for cluster \"cluster-nightly-4-23-vw9ss6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e0f2c0\u003e: \n failed to create NodePool np-vw9ss6: failed waiting for nodepool=\"np-vw9ss6\" for cluster \"cluster-nightly-4-23-vw9ss6\" in resourcegroup=\"rg-nightly-4-23-66zfk978pxch\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\",\n \"name\": \"04cd216e-5e0e-4e76-a1ab-6ea26fbb70a0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:42:21.716380971Z\",\n \"endTime\": \"2026-10-03T00:01:23.832580289Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-qhsr8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-wcqs9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-qhsr8: Creating: virtualmachine creating or updating\\nMachine h7d3c7e5o0i6e7s-np-vw9ss6-lw4g2-wcqs9: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c9bd3c052fc1970414f52ffb15a4cd3ebd9c34776d28d29908bf7e6dd35f2f8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"dfbe2f9b56ec8f589a40188c05855969f4988812afdbce974f8743866b87d6f5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rx699z\" for cluster \"cluster-nightly-5-1-rx699z\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb8200\u003e: \n failed to create NodePool np-rx699z: failed waiting for nodepool=\"np-rx699z\" for cluster \"cluster-nightly-5-1-rx699z\" in resourcegroup=\"rg-nightly-5-1-x6gpqlf2wmlk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ac35c1-1131-4bbe-9ed8-04f87748743e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ac35c1-1131-4bbe-9ed8-04f87748743e\",\n \"name\": \"13ac35c1-1131-4bbe-9ed8-04f87748743e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:40:19.915190927Z\",\n \"endTime\": \"2026-10-02T23:59:23.95618655Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-d28hs: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-kswmm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-d28hs: Creating: virtualmachine creating or updating\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-kswmm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rx699z\" for cluster \"cluster-nightly-5-1-rx699z\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb8200\u003e: \n failed to create NodePool np-rx699z: failed waiting for nodepool=\"np-rx699z\" for cluster \"cluster-nightly-5-1-rx699z\" in resourcegroup=\"rg-nightly-5-1-x6gpqlf2wmlk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ac35c1-1131-4bbe-9ed8-04f87748743e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/13ac35c1-1131-4bbe-9ed8-04f87748743e\",\n \"name\": \"13ac35c1-1131-4bbe-9ed8-04f87748743e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T23:40:19.915190927Z\",\n \"endTime\": \"2026-10-02T23:59:23.95618655Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-d28hs: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-kswmm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-d28hs: Creating: virtualmachine creating or updating\\nMachine t2t5i8t7n0a2i6q-np-rx699z-9wkrd-kswmm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cb3dfab3a1dc7e7676ad1d84e34d164aa7c286f8fc7cc2f6c813575fd3d25854","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"487336fbee3aa09886b1644e07daab686d89b729e7e6fdb636dc41d4196e0756","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001680330\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kcc2h87qnl8v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001680330\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-kcc2h87qnl8v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d5776c6abb183a3f6781d5f0c5eaec1cfac5327b9912de6d9615e6bd221f1a20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"aae832d22fd2e270ae046b4cd1453e86b9c22535700ac5df9912205534e9a89c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be25e0\u003e: \n failed to create NodePool np-f6fpxn: failed starting nodepool creation \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\" in resourcegroup=\"rg-nightly-4-20-9z2m2dsr8b8d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9z2m2dsr8b8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-f6fpxn/nodePools/np-f6fpxn\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000be25e0\u003e: \n failed to create NodePool np-f6fpxn: failed starting nodepool creation \"np-f6fpxn\" for cluster \"cluster-nightly-4-20-f6fpxn\" in resourcegroup=\"rg-nightly-4-20-9z2m2dsr8b8d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9z2m2dsr8b8d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-f6fpxn/nodePools/np-f6fpxn\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"db203ed7bd9d663eede6d203c87152d9249c020886743e960789f814e255841a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2106157461693009920","occurred_at":"2026-10-02T23:00:44Z","failed_at":"e2e","signature_id":"fc18079b3e233421ba88f574156106a7974d4521c46097383314a9ad391da2d6","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010a36c0\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-tsckb2km58j7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-tsckb2km58j7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010a36c0\u003e: \n failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-tsckb2km58j7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-tsckb2km58j7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"02985e80f9e6dd23af129635b4bd016a473a6a57b067e6f88bbc87e7f20a5595","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":25,"failure_pattern_ids":["02985e80f9e6dd23af129635b4bd016a473a6a57b067e6f88bbc87e7f20a5595","37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","40cded750dae4548d95aa35d7a8e08be199ce73f5c28df88f85a1ab20784f117","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":25,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T20:18:59Z","started_at":"2026-10-02T20:18:59Z","completed_at":"2026-10-02T22:33:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T22:48:12.876088187Z","timing_metadata_checked_at":"2026-10-02T22:48:12.876088187Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T22:48:12.879316476Z","region_metadata_checked_at":"2026-10-02T22:48:12.879316476Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"096475a34230de023ffdf18480e79da893971abf21fa152226c5177638e4acfd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"4b3072ea271a5fa9392b9001a31d9bdf8953c887295f5949247942b3ff4ae13e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a95460\u003e: \n failed to create NodePool np-k6xcnn: failed waiting for nodepool=\"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\" in resourcegroup=\"rg-candidate-5-1-xcbcjc2xvxnz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n \"name\": \"f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:05:55.776603892Z\",\n \"endTime\": \"2026-10-02T21:24:58.213196373Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a95460\u003e: \n failed to create NodePool np-k6xcnn: failed waiting for nodepool=\"np-k6xcnn\" for cluster \"cluster-candidate-5-1-k6xcnn\" in resourcegroup=\"rg-candidate-5-1-xcbcjc2xvxnz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n \"name\": \"f4a6f0d3-f2f8-4021-bdab-85f57fe8a1f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:05:55.776603892Z\",\n \"endTime\": \"2026-10-02T21:24:58.213196373Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"40c7fe4c3256bcb09f65c2b610653c09d7451653e66201f04740388781b0c131","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"d53697eeb27bf8ab55313c9278e2e56de2e878d8f994cba4d1ca76149b70e248","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001880060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-flb2g5s6xxjr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c3afe0f8-5077-4a5f-b1fe-943343d11f7a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c3afe0f8-5077-4a5f-b1fe-943343d11f7a\",\n \"name\": \"c3afe0f8-5077-4a5f-b1fe-943343d11f7a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:40:40.455432211Z\",\n \"endTime\": \"2026-10-02T20:59:49.144387256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-4kgx8-km75d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-4kgx8-wjql6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-4kgx8-km75d: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-4kgx8-wjql6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001880060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-flb2g5s6xxjr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c3afe0f8-5077-4a5f-b1fe-943343d11f7a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c3afe0f8-5077-4a5f-b1fe-943343d11f7a\",\n \"name\": \"c3afe0f8-5077-4a5f-b1fe-943343d11f7a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:40:40.455432211Z\",\n \"endTime\": \"2026-10-02T20:59:49.144387256Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-4kgx8-km75d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-4kgx8-wjql6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-4kgx8-km75d: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-4kgx8-wjql6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"49de21f41ff3ae1191602b48f25c5aa2e329b73b3beb517c5c54af7a669b45ed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"250d803c2dde1e52d60a32aaa2d7a8f52b97c3de220de17e241c3f6720b1dab2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rfrkgm\" for cluster \"cluster-candidate-4-22-rfrkgm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0efa0\u003e: \n failed to create NodePool np-rfrkgm: failed waiting for nodepool=\"np-rfrkgm\" for cluster \"cluster-candidate-4-22-rfrkgm\" in resourcegroup=\"rg-candidate-4-22-4dnctl8qsdmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd66a171-5708-426c-a085-dc7fbb337a5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd66a171-5708-426c-a085-dc7fbb337a5f\",\n \"name\": \"bd66a171-5708-426c-a085-dc7fbb337a5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:02:30.361632741Z\",\n \"endTime\": \"2026-10-02T21:21:38.229702877Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-dgcdt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-z5pzp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-dgcdt: Creating: virtualmachine creating or updating\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-z5pzp: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rfrkgm\" for cluster \"cluster-candidate-4-22-rfrkgm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a0efa0\u003e: \n failed to create NodePool np-rfrkgm: failed waiting for nodepool=\"np-rfrkgm\" for cluster \"cluster-candidate-4-22-rfrkgm\" in resourcegroup=\"rg-candidate-4-22-4dnctl8qsdmb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd66a171-5708-426c-a085-dc7fbb337a5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/bd66a171-5708-426c-a085-dc7fbb337a5f\",\n \"name\": \"bd66a171-5708-426c-a085-dc7fbb337a5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:02:30.361632741Z\",\n \"endTime\": \"2026-10-02T21:21:38.229702877Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-dgcdt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-z5pzp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-dgcdt: Creating: virtualmachine creating or updating\\nMachine z0t1i4f5k3c2g3w-np-rfrkgm-drz4w-z5pzp: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6bff5f7c3c8571a0d959d33c1b3491dc6e9e47b84700f612570d06237c20a164","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"2b86c91a6a4b689be874849e9832db0f4b59836029e6d1ac55b1fd9ebea622cc","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbb780\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-9v5cc2pldl4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\",\n \"name\": \"64c53475-70e9-4ead-ac9d-b255c14e111e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:37:10.986126021Z\",\n \"endTime\": \"2026-10-02T20:56:19.371811745Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:141]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbb780\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-9v5cc2pldl4b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/64c53475-70e9-4ead-ac9d-b255c14e111e\",\n \"name\": \"64c53475-70e9-4ead-ac9d-b255c14e111e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:37:10.986126021Z\",\n \"endTime\": \"2026-10-02T20:56:19.371811745Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-9v5cc2pldl4b--managed/providers/Microsoft.Compute/virtualMachines/v2z0r1j6g1i1v5p-np-1-pkgzx-qrzhh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7094f0d1bcc262750230fc7e73625955c623a7bfa01e60ad26f60f32c7406359","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"04ae0703900792edf19aad7f285f7ea250c1b64e19621d532ae6fed3c83dba96","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000c2b720\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fgzfw6brhrkb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n \"name\": \"6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:34:15.426377086Z\",\n \"endTime\": \"2026-10-02T21:53:18.348312497Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: Creating: virtualmachine creating or updating\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000c2b720\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-fgzfw6brhrkb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n \"name\": \"6270f4b8-a6c4-4356-a32a-feb00b0c9403\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:34:15.426377086Z\",\n \"endTime\": \"2026-10-02T21:53:18.348312497Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-8sclt: Creating: virtualmachine creating or updating\\nMachine p8k4o5c9o8v2d0a-np-deletable-9k5ld-ksnwd: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"7fc7074a2cc6bc51764b983c068b6184b7facdf76df5394955c08f82f22416f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"f1c205eb5adab7b42be8bb2e5810e6b3d460997323e06bef104b859fa7c1c1fc","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f04d60\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-j6tfrrstctwj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2237f1ea-36ac-4925-ab4c-3bf34b150641\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2237f1ea-36ac-4925-ab4c-3bf34b150641\",\n \"name\": \"2237f1ea-36ac-4925-ab4c-3bf34b150641\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:40:29.897644584Z\",\n \"endTime\": \"2026-10-02T20:59:29.402047255Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-krs6w-gdbz8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-krs6w-gdbz8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f04d60\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-j6tfrrstctwj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2237f1ea-36ac-4925-ab4c-3bf34b150641\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2237f1ea-36ac-4925-ab4c-3bf34b150641\",\n \"name\": \"2237f1ea-36ac-4925-ab4c-3bf34b150641\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:40:29.897644584Z\",\n \"endTime\": \"2026-10-02T20:59:29.402047255Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-krs6w-gdbz8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-krs6w-gdbz8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"809b3ca47a0465dfb83498ae46c9bace135a5d4f0c91d5c0ef4aa82ba3138f06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"557730ef8a8a538ac02f9730574bc105fe419054c1342595ab60e5a9b836a55b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qpmglj\" for cluster \"cluster-candidate-4-20-qpmglj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00128af20\u003e: \n failed to create NodePool np-qpmglj: failed waiting for nodepool=\"np-qpmglj\" for cluster \"cluster-candidate-4-20-qpmglj\" in resourcegroup=\"rg-candidate-4-20-lrjrfzqq46lq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f59c352-b15a-44fe-8eb1-052155b6abc3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f59c352-b15a-44fe-8eb1-052155b6abc3\",\n \"name\": \"8f59c352-b15a-44fe-8eb1-052155b6abc3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:01:08.060624103Z\",\n \"endTime\": \"2026-10-02T21:20:08.251497411Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-pngl6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-rh562: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-pngl6: Creating: virtualmachine creating or updating\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-rh562: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qpmglj\" for cluster \"cluster-candidate-4-20-qpmglj\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00128af20\u003e: \n failed to create NodePool np-qpmglj: failed waiting for nodepool=\"np-qpmglj\" for cluster \"cluster-candidate-4-20-qpmglj\" in resourcegroup=\"rg-candidate-4-20-lrjrfzqq46lq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f59c352-b15a-44fe-8eb1-052155b6abc3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8f59c352-b15a-44fe-8eb1-052155b6abc3\",\n \"name\": \"8f59c352-b15a-44fe-8eb1-052155b6abc3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:01:08.060624103Z\",\n \"endTime\": \"2026-10-02T21:20:08.251497411Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-pngl6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-rh562: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-pngl6: Creating: virtualmachine creating or updating\\nMachine a7s6c5x0k3i6u9a-np-qpmglj-tcqmc-rh562: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"997ea6840b66749ea36a8cd61c6485f99287d70f683f1fb6299a3abff3b69eba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"3ccc455f5000d71ad9adb6c80c8655900d46f92c09932844cfe825c30dea8f29","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001172048\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-5sg2h42pm8fv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n \"name\": \"4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:43:49.738242165Z\",\n \"endTime\": \"2026-10-02T21:02:49.398233298Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001172048\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-5sg2h42pm8fv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n \"name\": \"4ac0d101-3fd0-42ea-ae32-a6afba54d453\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:43:49.738242165Z\",\n \"endTime\": \"2026-10-02T21:02:49.398233298Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-wg9bd-twwb6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-wg9bd-z9q58: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"a4db154fd9c0a797d2debd5028cb127452841c9df00912c6d836549ef1804ad0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"abdd7c859f25fcc9e0e3eca9fc927795be2131c3357811538ef9ccf29668a7b0","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f6f0a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-cccnsncfs2r4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c87a1bea-d45c-40f0-b36e-76c1418b3181\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c87a1bea-d45c-40f0-b36e-76c1418b3181\",\n \"name\": \"c87a1bea-d45c-40f0-b36e-76c1418b3181\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:39:19.772177296Z\",\n \"endTime\": \"2026-10-02T20:58:19.197731682Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-mcr5z: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-pn7ft: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-mcr5z: Creating: virtualmachine creating or updating\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-pn7ft: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f6f0a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-cccnsncfs2r4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c87a1bea-d45c-40f0-b36e-76c1418b3181\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c87a1bea-d45c-40f0-b36e-76c1418b3181\",\n \"name\": \"c87a1bea-d45c-40f0-b36e-76c1418b3181\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T20:39:19.772177296Z\",\n \"endTime\": \"2026-10-02T20:58:19.197731682Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-mcr5z: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-pn7ft: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-mcr5z: Creating: virtualmachine creating or updating\\nMachine m9d7f6f9z0s9j7k-np-1-m4ndt-pn7ft: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d5504a933d38e17db69389c3b78d3c03fac9b8947587ff7c8791691a178db9ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"df71d9af24f4d024cb085b025d0e7702e69475d8342378c7ed71b98fdfda61a5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4rsvdk\" for cluster \"cluster-candidate-5-0-4rsvdk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ced660\u003e: \n failed to create NodePool np-4rsvdk: failed waiting for nodepool=\"np-4rsvdk\" for cluster \"cluster-candidate-5-0-4rsvdk\" in resourcegroup=\"rg-candidate-5-0-mr9dnz4m68mv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe99a24d-45b0-4c02-b399-8a58c33f760f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe99a24d-45b0-4c02-b399-8a58c33f760f\",\n \"name\": \"fe99a24d-45b0-4c02-b399-8a58c33f760f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:03:59.509943483Z\",\n \"endTime\": \"2026-10-02T21:23:08.337275316Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gjgrx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gkprg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gjgrx: Creating: virtualmachine creating or updating\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gkprg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-4rsvdk\" for cluster \"cluster-candidate-5-0-4rsvdk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ced660\u003e: \n failed to create NodePool np-4rsvdk: failed waiting for nodepool=\"np-4rsvdk\" for cluster \"cluster-candidate-5-0-4rsvdk\" in resourcegroup=\"rg-candidate-5-0-mr9dnz4m68mv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe99a24d-45b0-4c02-b399-8a58c33f760f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/fe99a24d-45b0-4c02-b399-8a58c33f760f\",\n \"name\": \"fe99a24d-45b0-4c02-b399-8a58c33f760f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:03:59.509943483Z\",\n \"endTime\": \"2026-10-02T21:23:08.337275316Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gjgrx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gkprg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gjgrx: Creating: virtualmachine creating or updating\\nMachine g6y4a6l5u7h6m2s-np-4rsvdk-jgm75-gkprg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d7ec549d4a5026535a9f1f3502a62dff432dfd2420632ef356b55fa7d09174b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"7ed334e95fee4ff7d071b052175951829e19af47fe9ec8e38db681d9e969f735","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d194a0\u003e: \n failed to create NodePool np-7lqqwg: failed waiting for nodepool=\"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\" in resourcegroup=\"rg-candidate-4-21-fsj5jdg9tljp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\",\n \"name\": \"d395f2bf-88ca-4f84-8998-c863978caf3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:03:18.868079791Z\",\n \"endTime\": \"2026-10-02T21:22:18.599909683Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d194a0\u003e: \n failed to create NodePool np-7lqqwg: failed waiting for nodepool=\"np-7lqqwg\" for cluster \"cluster-candidate-4-21-7lqqwg\" in resourcegroup=\"rg-candidate-4-21-fsj5jdg9tljp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d395f2bf-88ca-4f84-8998-c863978caf3f\",\n \"name\": \"d395f2bf-88ca-4f84-8998-c863978caf3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:03:18.868079791Z\",\n \"endTime\": \"2026-10-02T21:22:18.599909683Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ece2ff7cc8c9394af8ab3d38412290220d0120ec136fc1d6dafe1061e92efc4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106116759458156544","occurred_at":"2026-10-02T20:18:59Z","failed_at":"e2e","signature_id":"6a6e10696b7a19366e889f690b2ab374029b36eaa72533a7417b96b82b33497e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-878fr2\" for cluster \"cluster-candidate-4-23-878fr2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb09a0\u003e: \n failed to create NodePool np-878fr2: failed waiting for nodepool=\"np-878fr2\" for cluster \"cluster-candidate-4-23-878fr2\" in resourcegroup=\"rg-candidate-4-23-tq4tlj8sn8rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d39da2e3-e8ed-4ecb-a802-520caf1d680a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d39da2e3-e8ed-4ecb-a802-520caf1d680a\",\n \"name\": \"d39da2e3-e8ed-4ecb-a802-520caf1d680a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:05:40.949801842Z\",\n \"endTime\": \"2026-10-02T21:24:48.213673897Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-7czvr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-wcjsm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-7czvr: Creating: virtualmachine creating or updating\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-wcjsm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-878fr2\" for cluster \"cluster-candidate-4-23-878fr2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb09a0\u003e: \n failed to create NodePool np-878fr2: failed waiting for nodepool=\"np-878fr2\" for cluster \"cluster-candidate-4-23-878fr2\" in resourcegroup=\"rg-candidate-4-23-tq4tlj8sn8rt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d39da2e3-e8ed-4ecb-a802-520caf1d680a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d39da2e3-e8ed-4ecb-a802-520caf1d680a\",\n \"name\": \"d39da2e3-e8ed-4ecb-a802-520caf1d680a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T21:05:40.949801842Z\",\n \"endTime\": \"2026-10-02T21:24:48.213673897Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-7czvr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-wcjsm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-7czvr: Creating: virtualmachine creating or updating\\nMachine g4g6i1d1r4t2m8l-np-878fr2-58lpw-wcjsm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T17:45:08Z","started_at":"2026-10-02T17:45:08Z","completed_at":"2026-10-02T20:14:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T20:31:44.030568391Z","timing_metadata_checked_at":"2026-10-02T20:31:44.030568391Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T20:31:44.033696681Z","region_metadata_checked_at":"2026-10-02T20:31:44.033696681Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"06b39523714f8e8b897e0de9dba735e4495bd8afb0f523c0098db71c4a357721","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"f3c82f6a3a610eb2845b45bce009911813c9de13006d2bd6433c177db3846841","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee2740\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ht5gkd\" in resourcegroup=\"rg-np-noedge-ht5gkd-s9tjwqwzr9v2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2685dbe2-74b7-44fa-8866-0542656a2cbe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2685dbe2-74b7-44fa-8866-0542656a2cbe\",\n \"name\": \"2685dbe2-74b7-44fa-8866-0542656a2cbe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:26:52.969260057Z\",\n \"endTime\": \"2026-10-02T19:45:54.294162813Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-ht5gkd-s9tjwqwzr9v2-np-ne-ht5gkd--managed/providers/Microsoft.Compute/virtualMachines/s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-ht5gkd-s9tjwqwzr9v2-np-ne-ht5gkd--managed/providers/Microsoft.Compute/virtualMachines/s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ee2740\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-ht5gkd\" in resourcegroup=\"rg-np-noedge-ht5gkd-s9tjwqwzr9v2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2685dbe2-74b7-44fa-8866-0542656a2cbe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2685dbe2-74b7-44fa-8866-0542656a2cbe\",\n \"name\": \"2685dbe2-74b7-44fa-8866-0542656a2cbe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:26:52.969260057Z\",\n \"endTime\": \"2026-10-02T19:45:54.294162813Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-ht5gkd-s9tjwqwzr9v2-np-ne-ht5gkd--managed/providers/Microsoft.Compute/virtualMachines/s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-ht5gkd-s9tjwqwzr9v2-np-ne-ht5gkd--managed/providers/Microsoft.Compute/virtualMachines/s8l8q3n7s8a7r2q-npnoedge-4-20-gplmr-p7qxl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"305355cd1672855345c55c9cb833c281f5b35129e59e38a0342d0122286831a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"0f05a349b9fb3a19a653fac755019ca3729cea73f673115f377ea1ff53ca460a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-97jnb4\" for cluster \"cluster-candidate-4-22-97jnb4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a61960\u003e: \n failed to create NodePool np-97jnb4: failed waiting for nodepool=\"np-97jnb4\" for cluster \"cluster-candidate-4-22-97jnb4\" in resourcegroup=\"rg-candidate-4-22-gqvl8n7h4bmq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d14fd998-aa3e-49c8-a405-91cc8c02a377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d14fd998-aa3e-49c8-a405-91cc8c02a377\",\n \"name\": \"d14fd998-aa3e-49c8-a405-91cc8c02a377\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:49:32.578309403Z\",\n \"endTime\": \"2026-10-02T19:08:33.241090855Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-dkh4p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-hz2bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-dkh4p: Creating: virtualmachine creating or updating\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-hz2bh: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-97jnb4\" for cluster \"cluster-candidate-4-22-97jnb4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a61960\u003e: \n failed to create NodePool np-97jnb4: failed waiting for nodepool=\"np-97jnb4\" for cluster \"cluster-candidate-4-22-97jnb4\" in resourcegroup=\"rg-candidate-4-22-gqvl8n7h4bmq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d14fd998-aa3e-49c8-a405-91cc8c02a377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d14fd998-aa3e-49c8-a405-91cc8c02a377\",\n \"name\": \"d14fd998-aa3e-49c8-a405-91cc8c02a377\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:49:32.578309403Z\",\n \"endTime\": \"2026-10-02T19:08:33.241090855Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-dkh4p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-hz2bh: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-dkh4p: Creating: virtualmachine creating or updating\\nMachine v5r4w0l0u4k4n4f-np-97jnb4-k5gv8-hz2bh: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"57b017b159e60c7c24363451d6c46408f0d20e1ce34b2b43d09853443dad8f36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"23c88eb451a5cdd79574abe8ef9812935c0e0d4c234696f73be4ca6bce4d6bdc","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001522060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nl8vdbhxwtvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n \"name\": \"31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:27:19.35370587Z\",\n \"endTime\": \"2026-10-02T18:46:22.521838138Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001522060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-nl8vdbhxwtvx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n \"name\": \"31e50407-d006-4e42-9498-e3bbc2ee0de1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:27:19.35370587Z\",\n \"endTime\": \"2026-10-02T18:46:22.521838138Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-h6sjn-cnj2s: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-h6sjn-h8wr9: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"6a4feb2d0892c04953ef2c114e0b49c03fb8abe3c287f3e70a7cdd4dd7858f11","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"63e9790a996ec967d16e002977f953d1b88bb5b5420740a8c85e4a4ad4676d27","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kfvn25\" for cluster \"cluster-candidate-4-21-kfvn25\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b3b220\u003e: \n failed to create NodePool np-kfvn25: failed waiting for nodepool=\"np-kfvn25\" for cluster \"cluster-candidate-4-21-kfvn25\" in resourcegroup=\"rg-candidate-4-21-85wnrgqqbd2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b12e98e2-7f93-4004-87fa-b863b0f0044f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b12e98e2-7f93-4004-87fa-b863b0f0044f\",\n \"name\": \"b12e98e2-7f93-4004-87fa-b863b0f0044f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:49:07.875106668Z\",\n \"endTime\": \"2026-10-02T19:08:13.588598197Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-f65j4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-g5bl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-f65j4: Creating: virtualmachine creating or updating\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-g5bl8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kfvn25\" for cluster \"cluster-candidate-4-21-kfvn25\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b3b220\u003e: \n failed to create NodePool np-kfvn25: failed waiting for nodepool=\"np-kfvn25\" for cluster \"cluster-candidate-4-21-kfvn25\" in resourcegroup=\"rg-candidate-4-21-85wnrgqqbd2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b12e98e2-7f93-4004-87fa-b863b0f0044f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b12e98e2-7f93-4004-87fa-b863b0f0044f\",\n \"name\": \"b12e98e2-7f93-4004-87fa-b863b0f0044f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:49:07.875106668Z\",\n \"endTime\": \"2026-10-02T19:08:13.588598197Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-f65j4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-g5bl8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-f65j4: Creating: virtualmachine creating or updating\\nMachine l6p5f7e6i7t3n7m-np-kfvn25-4sz9p-g5bl8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7a70a53372f675de9d9be7c3a4f423e6c619c728d3dcba789f1f378cf1efba0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"f9e948ed90e8383bba7bd2c174ac649975b4723ac3ec4ecc8f0a4e9d83325283","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vvnbz4\" for cluster \"cluster-candidate-4-20-vvnbz4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c04c80\u003e: \n failed to create NodePool np-vvnbz4: failed waiting for nodepool=\"np-vvnbz4\" for cluster \"cluster-candidate-4-20-vvnbz4\" in resourcegroup=\"rg-candidate-4-20-6rrzp6smfk8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a84f10bd-1000-4ff5-b696-ec0471f172d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a84f10bd-1000-4ff5-b696-ec0471f172d5\",\n \"name\": \"a84f10bd-1000-4ff5-b696-ec0471f172d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:45:19.007012696Z\",\n \"endTime\": \"2026-10-02T19:04:23.174354297Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-jqrxn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-rd5xk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-jqrxn: Creating: virtualmachine creating or updating\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-rd5xk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-vvnbz4\" for cluster \"cluster-candidate-4-20-vvnbz4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c04c80\u003e: \n failed to create NodePool np-vvnbz4: failed waiting for nodepool=\"np-vvnbz4\" for cluster \"cluster-candidate-4-20-vvnbz4\" in resourcegroup=\"rg-candidate-4-20-6rrzp6smfk8r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a84f10bd-1000-4ff5-b696-ec0471f172d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a84f10bd-1000-4ff5-b696-ec0471f172d5\",\n \"name\": \"a84f10bd-1000-4ff5-b696-ec0471f172d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:45:19.007012696Z\",\n \"endTime\": \"2026-10-02T19:04:23.174354297Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-jqrxn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-rd5xk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-jqrxn: Creating: virtualmachine creating or updating\\nMachine j3l2b9y9v8f1o9h-np-vvnbz4-llscx-rd5xk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7e8a6452e1d5372e6a4a55264eda0a7050b31b32932b66ce044aa5cceef70926","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"94dbc4ea2092ec4f2334ecba4369c328eb3548419ec0eafdb59df728c566ad14","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qv99kb\" for cluster \"cluster-candidate-4-23-qv99kb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c78ba0\u003e: \n failed to create NodePool np-qv99kb: failed waiting for nodepool=\"np-qv99kb\" for cluster \"cluster-candidate-4-23-qv99kb\" in resourcegroup=\"rg-candidate-4-23-xhtfbmhw8nkt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93ec2ac0-8140-4df7-861f-8f5e69bd2825\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93ec2ac0-8140-4df7-861f-8f5e69bd2825\",\n \"name\": \"93ec2ac0-8140-4df7-861f-8f5e69bd2825\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:47:56.273213854Z\",\n \"endTime\": \"2026-10-02T19:07:03.234946252Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-nhvhz: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-xq4s5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-nhvhz: Creating: virtualmachine creating or updating\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-xq4s5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qv99kb\" for cluster \"cluster-candidate-4-23-qv99kb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c78ba0\u003e: \n failed to create NodePool np-qv99kb: failed waiting for nodepool=\"np-qv99kb\" for cluster \"cluster-candidate-4-23-qv99kb\" in resourcegroup=\"rg-candidate-4-23-xhtfbmhw8nkt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93ec2ac0-8140-4df7-861f-8f5e69bd2825\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/93ec2ac0-8140-4df7-861f-8f5e69bd2825\",\n \"name\": \"93ec2ac0-8140-4df7-861f-8f5e69bd2825\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:47:56.273213854Z\",\n \"endTime\": \"2026-10-02T19:07:03.234946252Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-nhvhz: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-xq4s5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-nhvhz: Creating: virtualmachine creating or updating\\nMachine o0z3a3e4t8t4l0z-np-qv99kb-rmmkl-xq4s5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"962349f04f77c21cfacbfb650160ee9b7d7b5c8a1ef11f41409afabe6c7bf8bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"ff0f4f85405eb058ab255730c9bfcc4dfebb9a6c55cbfe3ed6cc4e8bfb37dbfd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xf2nmx\" for cluster \"cluster-candidate-5-0-xf2nmx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000995320\u003e: \n failed to create NodePool np-xf2nmx: failed waiting for nodepool=\"np-xf2nmx\" for cluster \"cluster-candidate-5-0-xf2nmx\" in resourcegroup=\"rg-candidate-5-0-7jr6wdpsfvz4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\",\n \"name\": \"992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:49:51.032351879Z\",\n \"endTime\": \"2026-10-02T19:08:53.234722812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-kqt4r: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-nwmrg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-kqt4r: Creating: virtualmachine creating or updating\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-nwmrg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xf2nmx\" for cluster \"cluster-candidate-5-0-xf2nmx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000995320\u003e: \n failed to create NodePool np-xf2nmx: failed waiting for nodepool=\"np-xf2nmx\" for cluster \"cluster-candidate-5-0-xf2nmx\" in resourcegroup=\"rg-candidate-5-0-7jr6wdpsfvz4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\",\n \"name\": \"992a0c66-9c4a-4dc5-8b0f-2c46d65526fe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:49:51.032351879Z\",\n \"endTime\": \"2026-10-02T19:08:53.234722812Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-kqt4r: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-nwmrg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-kqt4r: Creating: virtualmachine creating or updating\\nMachine b1t7y5a4d6l6s5p-np-xf2nmx-bnzwr-nwmrg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"96aedd7bfda0362383c718c4ea76002c1bb06fb6573d1a07bf460124da373926","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"1991e4120c3a9404f4a08486e54a3b094270ef79c075deb389ccb4f87dd19186","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcce00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-kndcxqxwg5m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9940d584-67e5-4b2a-939b-305a5a782054\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9940d584-67e5-4b2a-939b-305a5a782054\",\n \"name\": \"9940d584-67e5-4b2a-939b-305a5a782054\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:26:54.435762769Z\",\n \"endTime\": \"2026-10-02T18:46:02.862827665Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-6stxx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-fvklg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-6stxx: Creating: virtualmachine creating or updating\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-fvklg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fcce00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-kndcxqxwg5m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9940d584-67e5-4b2a-939b-305a5a782054\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9940d584-67e5-4b2a-939b-305a5a782054\",\n \"name\": \"9940d584-67e5-4b2a-939b-305a5a782054\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:26:54.435762769Z\",\n \"endTime\": \"2026-10-02T18:46:02.862827665Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-6stxx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-fvklg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-6stxx: Creating: virtualmachine creating or updating\\nMachine l1e8j9h2p9p7t8t-np-1-dsx99-fvklg: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"afb7c28584b2497ec163cbad4f11b2f5049c464b9c15a9992bc4100ad2f35ea8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"c795eac422f14a1d4cf2c9186a4a74089ac3865e0cf6aad349163a72b03db067","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lvw97g\" for cluster \"cluster-candidate-5-1-lvw97g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e66f00\u003e: \n failed to create NodePool np-lvw97g: failed waiting for nodepool=\"np-lvw97g\" for cluster \"cluster-candidate-5-1-lvw97g\" in resourcegroup=\"rg-candidate-5-1-qrgrjlf6rkm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4a97c7bb-57af-45b7-8fb7-5b981cf7570f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4a97c7bb-57af-45b7-8fb7-5b981cf7570f\",\n \"name\": \"4a97c7bb-57af-45b7-8fb7-5b981cf7570f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:50:51.431353811Z\",\n \"endTime\": \"2026-10-02T19:09:53.217243745Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-95ljv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-cfkc9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-95ljv: Creating: virtualmachine creating or updating\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-cfkc9: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lvw97g\" for cluster \"cluster-candidate-5-1-lvw97g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e66f00\u003e: \n failed to create NodePool np-lvw97g: failed waiting for nodepool=\"np-lvw97g\" for cluster \"cluster-candidate-5-1-lvw97g\" in resourcegroup=\"rg-candidate-5-1-qrgrjlf6rkm9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4a97c7bb-57af-45b7-8fb7-5b981cf7570f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4a97c7bb-57af-45b7-8fb7-5b981cf7570f\",\n \"name\": \"4a97c7bb-57af-45b7-8fb7-5b981cf7570f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:50:51.431353811Z\",\n \"endTime\": \"2026-10-02T19:09:53.217243745Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-95ljv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-cfkc9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-95ljv: Creating: virtualmachine creating or updating\\nMachine l3n7x5q5h0o7l6w-np-lvw97g-9smn8-cfkc9: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b0e4cbb8c8cd579f631dab696b910ba233dafd020215195a0d44279abd496b37","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"d96bba282a27aaa6fbc92c0663dfd8f01103b1e67a58a200c8d1a2d1b8e4e832","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c9c280\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-gg8s9j\" in resourcegroup=\"rg-np-downgrade-gg8s9j-fnkpnt9xd47t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\",\n \"name\": \"a4186f06-3e4f-4125-8780-405aa920bd3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:29:21.050247186Z\",\n \"endTime\": \"2026-10-02T19:48:24.533281016Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c9c280\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-gg8s9j\" in resourcegroup=\"rg-np-downgrade-gg8s9j-fnkpnt9xd47t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a4186f06-3e4f-4125-8780-405aa920bd3f\",\n \"name\": \"a4186f06-3e4f-4125-8780-405aa920bd3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:29:21.050247186Z\",\n \"endTime\": \"2026-10-02T19:48:24.533281016Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-gg8s9j-fnkpnt9xd47t-np-dg-gg8s9j--managed/providers/Microsoft.Compute/virtualMachines/a7o8k0t2r9e8d6s-npdg-4-21-xzxrt-5htvd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ccf7cf35d852b5a080806ae3c01fd13af18181e686ea23cca7840c454a70b752","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"7f3920bc72425d6da0c2994e526b54c2ba9fd9d44a6c8cb1fdea88ed4dae4ca0","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001085220\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-dk96qxfcglkz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a2c0d616-d3e1-4467-a097-edfb164f91f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a2c0d616-d3e1-4467-a097-edfb164f91f8\",\n \"name\": \"a2c0d616-d3e1-4467-a097-edfb164f91f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:26:30.147930745Z\",\n \"endTime\": \"2026-10-02T18:45:32.745630715Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-jbbww-kglw7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-jbbww-kglw7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001085220\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-dk96qxfcglkz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a2c0d616-d3e1-4467-a097-edfb164f91f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a2c0d616-d3e1-4467-a097-edfb164f91f8\",\n \"name\": \"a2c0d616-d3e1-4467-a097-edfb164f91f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:26:30.147930745Z\",\n \"endTime\": \"2026-10-02T18:45:32.745630715Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-jbbww-kglw7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-jbbww-kglw7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"df313438a992a9865e2096234146bb071b0155b14d8f0c1c25d1b9026c360484","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"1e6f9fa59cd98bfc284387b5af1ed56122fcb8e577d42d119bd60d5ef8e20605","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00341d8a0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8s9pgvgwlgbj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n \"name\": \"e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:20:19.695876004Z\",\n \"endTime\": \"2026-10-02T19:39:24.271020967Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: Creating: virtualmachine creating or updating\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00341d8a0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-8s9pgvgwlgbj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n \"name\": \"e4e9a75c-b715-4a5f-8f78-888a4044e76e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T19:20:19.695876004Z\",\n \"endTime\": \"2026-10-02T19:39:24.271020967Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-4mb96: Creating: virtualmachine creating or updating\\nMachine k8i8k9n2m6a4a0n-np-delete-main-pkm89-8f5lf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"e0481009c3ee19e3b6efac1ad603e143030ff4cb3f5b1ca21eb16f4696b9d31f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2106078041405919232","occurred_at":"2026-10-02T17:45:08Z","failed_at":"e2e","signature_id":"3beeb059ed2eee1c225e5fa6e52e389772213e9dbefa4285fead2b9c61d38ee4","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001f12060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5x984bppgxmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f889fa2b-1853-4b53-b0a5-8af8bb1664c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f889fa2b-1853-4b53-b0a5-8af8bb1664c4\",\n \"name\": \"f889fa2b-1853-4b53-b0a5-8af8bb1664c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:28:21.353402223Z\",\n \"endTime\": \"2026-10-02T18:47:22.523072731Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-rvbvq-47m2n: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-rvbvq-dncqc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-rvbvq-47m2n: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-rvbvq-dncqc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001f12060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5x984bppgxmk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f889fa2b-1853-4b53-b0a5-8af8bb1664c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f889fa2b-1853-4b53-b0a5-8af8bb1664c4\",\n \"name\": \"f889fa2b-1853-4b53-b0a5-8af8bb1664c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T18:28:21.353402223Z\",\n \"endTime\": \"2026-10-02T18:47:22.523072731Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-rvbvq-47m2n: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-rvbvq-dncqc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-rvbvq-47m2n: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-rvbvq-dncqc: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T07:30:54Z","started_at":"2026-10-02T07:30:54Z","completed_at":"2026-10-02T09:51:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T10:00:10.31942121Z","timing_metadata_checked_at":"2026-10-02T10:00:10.31942121Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T10:00:10.322654298Z","region_metadata_checked_at":"2026-10-02T10:00:10.322654298Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"1491c913fd3907211e6e3fadfd6ec8cf83a8d0a9bbdcd73c6b7f5ebd107a8b8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"417e38533ea85fbd583e5552fc77a77554d65e5a8dd9d15343bf93a32380c555","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9d580\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-vrplznt5v54r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85d24adc-6f21-47c1-8812-f58f8001bb49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85d24adc-6f21-47c1-8812-f58f8001bb49\",\n \"name\": \"85d24adc-6f21-47c1-8812-f58f8001bb49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:56:13.127412921Z\",\n \"endTime\": \"2026-10-02T08:15:18.247774488Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-t8dkq-6vlv4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-t8dkq-6vlv4: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9d580\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-vrplznt5v54r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85d24adc-6f21-47c1-8812-f58f8001bb49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85d24adc-6f21-47c1-8812-f58f8001bb49\",\n \"name\": \"85d24adc-6f21-47c1-8812-f58f8001bb49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:56:13.127412921Z\",\n \"endTime\": \"2026-10-02T08:15:18.247774488Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-t8dkq-6vlv4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-t8dkq-6vlv4: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3add730a4d0f461956b699cfb69439560ad92350a3f3abae2844261da6b72dcf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"8ae482f04181781883adcf3137eae2256182da0fa9081ef6d8de86debd8ad010","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000666c60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bgj7nclctwxw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\",\n \"name\": \"29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:52:01.290279416Z\",\n \"endTime\": \"2026-10-02T09:11:09.436811147Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-4k649: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bgj7nclctwxw--managed/providers/Microsoft.Compute/virtualMachines/h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-4k649: Creating: virtualmachine creating or updating\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bgj7nclctwxw--managed/providers/Microsoft.Compute/virtualMachines/h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000666c60\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-bgj7nclctwxw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\",\n \"name\": \"29fe846e-0932-41fe-a2dd-ff17d6cc5ac0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:52:01.290279416Z\",\n \"endTime\": \"2026-10-02T09:11:09.436811147Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-4k649: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bgj7nclctwxw--managed/providers/Microsoft.Compute/virtualMachines/h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-4k649: Creating: virtualmachine creating or updating\\nMachine h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-bgj7nclctwxw--managed/providers/Microsoft.Compute/virtualMachines/h3r3f2u9k1p8i1c-np-delete-main-hwgw9-p69q6 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"434022071ee4c10aaca9bb6a368811b5ee3ecd1fb7e0735c94eb8f09b2cf6ba2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"fdc0e08684b498bbfb00d4bb852da4d3016c65a4ec84f977294ad7bdbf25c4a1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-q645ss\" for cluster \"cluster-candidate-4-23-q645ss\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00147edc0\u003e: \n failed to create NodePool np-q645ss: failed waiting for nodepool=\"np-q645ss\" for cluster \"cluster-candidate-4-23-q645ss\" in resourcegroup=\"rg-candidate-4-23-97lbmpnc95ml\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\",\n \"name\": \"e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:22:25.175276447Z\",\n \"endTime\": \"2026-10-02T08:41:25.163899948Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-b9h5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-rcd4l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-b9h5q: Creating: virtualmachine creating or updating\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-rcd4l: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-q645ss\" for cluster \"cluster-candidate-4-23-q645ss\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00147edc0\u003e: \n failed to create NodePool np-q645ss: failed waiting for nodepool=\"np-q645ss\" for cluster \"cluster-candidate-4-23-q645ss\" in resourcegroup=\"rg-candidate-4-23-97lbmpnc95ml\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\",\n \"name\": \"e7c6ae1c-be71-48cb-b2ba-de56bb6c16d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:22:25.175276447Z\",\n \"endTime\": \"2026-10-02T08:41:25.163899948Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-b9h5q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-rcd4l: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-b9h5q: Creating: virtualmachine creating or updating\\nMachine x4b4k7o1x7w4m8z-np-q645ss-zvxb8-rcd4l: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"45c195e887f303240824afdd140fc3d19d456ebc8956d20da34436bb61fb655d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"4bb202a926156d42d840b7a9818cb5b7935514c66bb61eb6995e5c5d73101ec2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea0360\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5w2whf62mkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd107ed-298a-4008-9f25-801165f1c37e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd107ed-298a-4008-9f25-801165f1c37e\",\n \"name\": \"ffd107ed-298a-4008-9f25-801165f1c37e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:56:52.717139839Z\",\n \"endTime\": \"2026-10-02T08:15:58.568270167Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5w2whf62mkf6--managed/providers/Microsoft.Compute/virtualMachines/v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5w2whf62mkf6--managed/providers/Microsoft.Compute/virtualMachines/v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea0360\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-5w2whf62mkf6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd107ed-298a-4008-9f25-801165f1c37e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ffd107ed-298a-4008-9f25-801165f1c37e\",\n \"name\": \"ffd107ed-298a-4008-9f25-801165f1c37e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:56:52.717139839Z\",\n \"endTime\": \"2026-10-02T08:15:58.568270167Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5w2whf62mkf6--managed/providers/Microsoft.Compute/virtualMachines/v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5w2whf62mkf6--managed/providers/Microsoft.Compute/virtualMachines/v5b2u0b6b5e2c6j-np-1-rtdvc-2drvl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"47d20a86aa1a33efb166b16b730c3bde80133830b4a727cf87b9e75916dd82e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"615a1a8b96dee5eb1bf6319d009a9ff736c86e855049cffe408293ba6acfef53","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kf44kx\" for cluster \"cluster-candidate-4-22-kf44kx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001670be0\u003e: \n failed to create NodePool np-kf44kx: failed waiting for nodepool=\"np-kf44kx\" for cluster \"cluster-candidate-4-22-kf44kx\" in resourcegroup=\"rg-candidate-4-22-p972hm7v767t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f5cc41ef-2098-4080-80d0-52d4a6c605b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f5cc41ef-2098-4080-80d0-52d4a6c605b2\",\n \"name\": \"f5cc41ef-2098-4080-80d0-52d4a6c605b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:24:18.844576711Z\",\n \"endTime\": \"2026-10-02T08:43:25.00034882Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-ff2wq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-p7lf6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-ff2wq: Creating: virtualmachine creating or updating\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-p7lf6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kf44kx\" for cluster \"cluster-candidate-4-22-kf44kx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001670be0\u003e: \n failed to create NodePool np-kf44kx: failed waiting for nodepool=\"np-kf44kx\" for cluster \"cluster-candidate-4-22-kf44kx\" in resourcegroup=\"rg-candidate-4-22-p972hm7v767t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f5cc41ef-2098-4080-80d0-52d4a6c605b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f5cc41ef-2098-4080-80d0-52d4a6c605b2\",\n \"name\": \"f5cc41ef-2098-4080-80d0-52d4a6c605b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:24:18.844576711Z\",\n \"endTime\": \"2026-10-02T08:43:25.00034882Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-ff2wq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-p7lf6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-ff2wq: Creating: virtualmachine creating or updating\\nMachine j5o2i4r2x6i1i6v-np-kf44kx-5lnfn-p7lf6: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"69668cb5819485ec95ab221da8e51cc7b89985fbaa7337ca405d514f206ca508","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"e764784e704e2e2b27d861fbb0e9e4e39307c4efc2358883f5a0a0bfaf1ece26","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qbwjq7\" for cluster \"cluster-candidate-5-1-qbwjq7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f28640\u003e: \n failed to create NodePool np-qbwjq7: failed waiting for nodepool=\"np-qbwjq7\" for cluster \"cluster-candidate-5-1-qbwjq7\" in resourcegroup=\"rg-candidate-5-1-cxt8gqtcm56h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/411adf6f-822a-4f98-bc7d-64851f61532c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/411adf6f-822a-4f98-bc7d-64851f61532c\",\n \"name\": \"411adf6f-822a-4f98-bc7d-64851f61532c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:23:47.400908917Z\",\n \"endTime\": \"2026-10-02T08:42:55.012095583Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-dhvsm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-zlljx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-dhvsm: Creating: virtualmachine creating or updating\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-zlljx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-qbwjq7\" for cluster \"cluster-candidate-5-1-qbwjq7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f28640\u003e: \n failed to create NodePool np-qbwjq7: failed waiting for nodepool=\"np-qbwjq7\" for cluster \"cluster-candidate-5-1-qbwjq7\" in resourcegroup=\"rg-candidate-5-1-cxt8gqtcm56h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/411adf6f-822a-4f98-bc7d-64851f61532c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/411adf6f-822a-4f98-bc7d-64851f61532c\",\n \"name\": \"411adf6f-822a-4f98-bc7d-64851f61532c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:23:47.400908917Z\",\n \"endTime\": \"2026-10-02T08:42:55.012095583Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-dhvsm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-zlljx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-dhvsm: Creating: virtualmachine creating or updating\\nMachine x5o3w3y8k8n9n2p-np-qbwjq7-vv8c8-zlljx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"76bb947cd003eb4cd213e3dc50e7082655c5e22c2f3b96bd9622950d593acaf8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"d25ebb7feb22db43b9925f83b8d322fdcc4c54a5432fbc9ce5efb6aae6f0e97b","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000e761b0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-dwl8n7k5d292\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n \"name\": \"0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:58:36.615555909Z\",\n \"endTime\": \"2026-10-02T08:17:38.532193005Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc000e761b0\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-dwl8n7k5d292\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n \"name\": \"0cd68833-1cd8-4477-9f36-56a0fe7d00ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:58:36.615555909Z\",\n \"endTime\": \"2026-10-02T08:17:38.532193005Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-c9phq-b96vq: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-c9phq-gw2rm: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"77c04b42c3abea3eeb95b33bf011bf1cf93dc9673e81f284e4d654d077fd6692","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"6c3e750da0e9592b1f2dec07e4441b5b99e79bcaca5632e733e0724a36755aaf","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008aece0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-plwp75qbhvcq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d68804b-aa1f-441d-9c44-b697ee5be0e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d68804b-aa1f-441d-9c44-b697ee5be0e1\",\n \"name\": \"0d68804b-aa1f-441d-9c44-b697ee5be0e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:58:27.095320904Z\",\n \"endTime\": \"2026-10-02T08:17:28.456252233Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-j6stv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-jcmz5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-j6stv: Creating: virtualmachine creating or updating\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-jcmz5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008aece0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-plwp75qbhvcq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d68804b-aa1f-441d-9c44-b697ee5be0e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0d68804b-aa1f-441d-9c44-b697ee5be0e1\",\n \"name\": \"0d68804b-aa1f-441d-9c44-b697ee5be0e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:58:27.095320904Z\",\n \"endTime\": \"2026-10-02T08:17:28.456252233Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-j6stv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-jcmz5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-j6stv: Creating: virtualmachine creating or updating\\nMachine v5t1z8u9w4x9o4c-np-1-qj5gs-jcmz5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9fd7b20df5e7be1c66779b9074e9d22fb23832a595df836266165ee8c1132e6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"04d1cb3cb4234161cc07dedd18e19e98c569ee86bfc6884e682791a6475e5f03","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-65j846\" for cluster \"cluster-candidate-4-21-65j846\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd8540\u003e: \n failed to create NodePool np-65j846: failed waiting for nodepool=\"np-65j846\" for cluster \"cluster-candidate-4-21-65j846\" in resourcegroup=\"rg-candidate-4-21-8dj7spdjvcps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4b395e6-2b88-4e54-814b-bcbf478cdffc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4b395e6-2b88-4e54-814b-bcbf478cdffc\",\n \"name\": \"d4b395e6-2b88-4e54-814b-bcbf478cdffc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:23:11.897579976Z\",\n \"endTime\": \"2026-10-02T08:42:15.234899325Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-dczz6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-hp4tn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-dczz6: Creating: virtualmachine creating or updating\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-hp4tn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-65j846\" for cluster \"cluster-candidate-4-21-65j846\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dd8540\u003e: \n failed to create NodePool np-65j846: failed waiting for nodepool=\"np-65j846\" for cluster \"cluster-candidate-4-21-65j846\" in resourcegroup=\"rg-candidate-4-21-8dj7spdjvcps\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4b395e6-2b88-4e54-814b-bcbf478cdffc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d4b395e6-2b88-4e54-814b-bcbf478cdffc\",\n \"name\": \"d4b395e6-2b88-4e54-814b-bcbf478cdffc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:23:11.897579976Z\",\n \"endTime\": \"2026-10-02T08:42:15.234899325Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-dczz6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-hp4tn: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-dczz6: Creating: virtualmachine creating or updating\\nMachine p4r1l1y9v2e2n2b-np-65j846-2t8dj-hp4tn: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"db267170cc53eb6b28a1be1aacea12ce6ef663f1fc63f93670e625ebf2b424a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"ee727effd5d95b42b1f21ecbaf21ff986de2285544c181405cc38338639d85a7","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002194198\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kkklwf69d6rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6596056-a431-4061-9fcc-922385eb88b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6596056-a431-4061-9fcc-922385eb88b7\",\n \"name\": \"f6596056-a431-4061-9fcc-922385eb88b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:58:26.598790316Z\",\n \"endTime\": \"2026-10-02T08:17:28.2532697Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-t9npj-5lpcd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-t9npj-z8x9w: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-t9npj-5lpcd: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-t9npj-z8x9w: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002194198\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-kkklwf69d6rr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6596056-a431-4061-9fcc-922385eb88b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f6596056-a431-4061-9fcc-922385eb88b7\",\n \"name\": \"f6596056-a431-4061-9fcc-922385eb88b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T07:58:26.598790316Z\",\n \"endTime\": \"2026-10-02T08:17:28.2532697Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-t9npj-5lpcd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-t9npj-z8x9w: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-t9npj-5lpcd: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-t9npj-z8x9w: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dfd01cc5b407694b7f2d98a60e32845a0d41aa8f4609e7d7cecb9c2939272dbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"aa9cf2d4709f42c9eb5dad95a56141738554f1b7d752de0d4b0ca863ed35f08d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-96gwgz\" for cluster \"cluster-candidate-5-0-96gwgz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf2260\u003e: \n failed to create NodePool np-96gwgz: failed waiting for nodepool=\"np-96gwgz\" for cluster \"cluster-candidate-5-0-96gwgz\" in resourcegroup=\"rg-candidate-5-0-84kwhzj8r7w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7df18c5-1f74-4f58-93d8-951ddd66b332\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7df18c5-1f74-4f58-93d8-951ddd66b332\",\n \"name\": \"c7df18c5-1f74-4f58-93d8-951ddd66b332\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:23:19.198498666Z\",\n \"endTime\": \"2026-10-02T08:42:25.18491611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-4frj5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-d5fmk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-4frj5: Creating: virtualmachine creating or updating\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-d5fmk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-96gwgz\" for cluster \"cluster-candidate-5-0-96gwgz\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf2260\u003e: \n failed to create NodePool np-96gwgz: failed waiting for nodepool=\"np-96gwgz\" for cluster \"cluster-candidate-5-0-96gwgz\" in resourcegroup=\"rg-candidate-5-0-84kwhzj8r7w4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7df18c5-1f74-4f58-93d8-951ddd66b332\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c7df18c5-1f74-4f58-93d8-951ddd66b332\",\n \"name\": \"c7df18c5-1f74-4f58-93d8-951ddd66b332\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:23:19.198498666Z\",\n \"endTime\": \"2026-10-02T08:42:25.18491611Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-4frj5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-d5fmk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-4frj5: Creating: virtualmachine creating or updating\\nMachine m9p3c0m3i2a9q0y-np-96gwgz-x5zk7-d5fmk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f9e478368f37ecd0b1347af680ab6aa57f344170bbde5a159c6b1b05b5e2e239","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105923464547274752","occurred_at":"2026-10-02T07:30:54Z","failed_at":"e2e","signature_id":"de872c4f9bed78e595aa0309be8b2145ae00fa864cdae74c62b196f02e7a7c68","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tgr2wf\" for cluster \"cluster-candidate-4-20-tgr2wf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e1d940\u003e: \n failed to create NodePool np-tgr2wf: failed waiting for nodepool=\"np-tgr2wf\" for cluster \"cluster-candidate-4-20-tgr2wf\" in resourcegroup=\"rg-candidate-4-20-r2lrfp2tbsqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0734784e-738a-4b64-ad9c-e373ea6e9682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0734784e-738a-4b64-ad9c-e373ea6e9682\",\n \"name\": \"0734784e-738a-4b64-ad9c-e373ea6e9682\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:23:20.483388473Z\",\n \"endTime\": \"2026-10-02T08:42:25.166235902Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-jmm8m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-k7sxl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-jmm8m: Creating: virtualmachine creating or updating\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-k7sxl: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tgr2wf\" for cluster \"cluster-candidate-4-20-tgr2wf\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e1d940\u003e: \n failed to create NodePool np-tgr2wf: failed waiting for nodepool=\"np-tgr2wf\" for cluster \"cluster-candidate-4-20-tgr2wf\" in resourcegroup=\"rg-candidate-4-20-r2lrfp2tbsqc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0734784e-738a-4b64-ad9c-e373ea6e9682\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/0734784e-738a-4b64-ad9c-e373ea6e9682\",\n \"name\": \"0734784e-738a-4b64-ad9c-e373ea6e9682\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T08:23:20.483388473Z\",\n \"endTime\": \"2026-10-02T08:42:25.166235902Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-jmm8m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-k7sxl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-jmm8m: Creating: virtualmachine creating or updating\\nMachine x6z3k1g0g0y1m3q-np-tgr2wf-4vbp4-k7sxl: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T02:00:29Z","started_at":"2026-10-02T02:00:29Z","completed_at":"2026-10-02T05:11:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T05:27:48.820525766Z","timing_metadata_checked_at":"2026-10-02T05:27:48.820525766Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T05:27:48.823897156Z","region_metadata_checked_at":"2026-10-02T05:27:48.823897156Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"07b4e87bab48a73502f78d5af689f69fe1f8ed7ff9f6b86c2e522a5a4c796d20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"8712867601ac02520c8073037f11212160d4494ca873f5f46983b954dae750d9","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001276700\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d53fa45c-a9c4-43a3-b87a-c16df786eeb4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d53fa45c-a9c4-43a3-b87a-c16df786eeb4\",\n \"name\": \"d53fa45c-a9c4-43a3-b87a-c16df786eeb4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T04:09:42.171003862Z\",\n \"endTime\": \"2026-10-02T04:28:46.827990048Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-n4l4p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-qtmzk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-n4l4p: Creating: virtualmachine creating or updating\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-qtmzk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001276700\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d53fa45c-a9c4-43a3-b87a-c16df786eeb4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/d53fa45c-a9c4-43a3-b87a-c16df786eeb4\",\n \"name\": \"d53fa45c-a9c4-43a3-b87a-c16df786eeb4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T04:09:42.171003862Z\",\n \"endTime\": \"2026-10-02T04:28:46.827990048Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-n4l4p: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-qtmzk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-n4l4p: Creating: virtualmachine creating or updating\\nMachine y8h6g1h7a1t1v0i-np-deletable-c6pz2-qtmzk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}},{"row_id":"1fae92c6e879f21af0a90c9b3f9e558595dc88652f8c5c2c8eb4359cc930a059","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"5976bd54ef1912d8a56fae856cd640929ca801e0b38df661fe97065f1e1ddd1e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9rfwql\" for cluster \"cluster-candidate-5-1-9rfwql\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001103040\u003e: \n failed to create NodePool np-9rfwql: failed waiting for nodepool=\"np-9rfwql\" for cluster \"cluster-candidate-5-1-9rfwql\" in resourcegroup=\"rg-candidate-5-1-jznzltj2rxxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fd61fa-5104-45ad-bf92-0d32ab5a44e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fd61fa-5104-45ad-bf92-0d32ab5a44e5\",\n \"name\": \"34fd61fa-5104-45ad-bf92-0d32ab5a44e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:44:17.773093204Z\",\n \"endTime\": \"2026-10-02T04:03:23.679415683Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-9ptnl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-gscjf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-9ptnl: Creating: virtualmachine creating or updating\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-gscjf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9rfwql\" for cluster \"cluster-candidate-5-1-9rfwql\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001103040\u003e: \n failed to create NodePool np-9rfwql: failed waiting for nodepool=\"np-9rfwql\" for cluster \"cluster-candidate-5-1-9rfwql\" in resourcegroup=\"rg-candidate-5-1-jznzltj2rxxg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fd61fa-5104-45ad-bf92-0d32ab5a44e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/34fd61fa-5104-45ad-bf92-0d32ab5a44e5\",\n \"name\": \"34fd61fa-5104-45ad-bf92-0d32ab5a44e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:44:17.773093204Z\",\n \"endTime\": \"2026-10-02T04:03:23.679415683Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-9ptnl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-gscjf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-9ptnl: Creating: virtualmachine creating or updating\\nMachine s2j2b3a4i7k6f9u-np-9rfwql-2xh25-gscjf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"22f9c91ba521286f1112399191933a96daafb6f31cd806329d9f52862c0547cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"91fd4fecbd77ed6f997de413f52e960fcad2e588e3fa8f18e505fb83e8c11564","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002984078\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4httpsjkvppc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/27c9b728-c34e-4afa-93f4-b5118810ce82\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/27c9b728-c34e-4afa-93f4-b5118810ce82\",\n \"name\": \"27c9b728-c34e-4afa-93f4-b5118810ce82\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:19:57.766414352Z\",\n \"endTime\": \"2026-10-02T03:39:03.235272547Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-lszrc-8znc6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-lszrc-nxxm5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-lszrc-8znc6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-lszrc-nxxm5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002984078\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4httpsjkvppc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/27c9b728-c34e-4afa-93f4-b5118810ce82\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/27c9b728-c34e-4afa-93f4-b5118810ce82\",\n \"name\": \"27c9b728-c34e-4afa-93f4-b5118810ce82\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:19:57.766414352Z\",\n \"endTime\": \"2026-10-02T03:39:03.235272547Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-lszrc-8znc6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-lszrc-nxxm5: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-lszrc-8znc6: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-lszrc-nxxm5: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"445c691fc32c05ab1b8738556fb7f1484753f873c095e2cfd4b6c6772517b5d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"297f54696f0c10752edca7d9cf5c5d2f98707884ddccf8ef2f952e32d9f597c8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fs2tl4\" for cluster \"cluster-candidate-4-22-fs2tl4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dde140\u003e: \n failed to create NodePool np-fs2tl4: failed waiting for nodepool=\"np-fs2tl4\" for cluster \"cluster-candidate-4-22-fs2tl4\" in resourcegroup=\"rg-candidate-4-22-jzw6chr4d7m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/af47af07-70e2-47bd-9306-0c7f478d186e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/af47af07-70e2-47bd-9306-0c7f478d186e\",\n \"name\": \"af47af07-70e2-47bd-9306-0c7f478d186e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:39:14.039068534Z\",\n \"endTime\": \"2026-10-02T03:58:23.505666093Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-hwrpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-mh58m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-hwrpc: Creating: virtualmachine creating or updating\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-mh58m: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fs2tl4\" for cluster \"cluster-candidate-4-22-fs2tl4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dde140\u003e: \n failed to create NodePool np-fs2tl4: failed waiting for nodepool=\"np-fs2tl4\" for cluster \"cluster-candidate-4-22-fs2tl4\" in resourcegroup=\"rg-candidate-4-22-jzw6chr4d7m4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/af47af07-70e2-47bd-9306-0c7f478d186e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/af47af07-70e2-47bd-9306-0c7f478d186e\",\n \"name\": \"af47af07-70e2-47bd-9306-0c7f478d186e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:39:14.039068534Z\",\n \"endTime\": \"2026-10-02T03:58:23.505666093Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-hwrpc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-mh58m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-hwrpc: Creating: virtualmachine creating or updating\\nMachine g1v3q3i3v9q4u8m-np-fs2tl4-ttfnq-mh58m: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ef4b5578883c7582d166104a0a47c734f8dadf2a4ee097a0b0af9768ff6f4e9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"7c052020cd8edffb258e72da7eba5bdb0cf97b0d362af401b3cb610f8b906cf6","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000943f60\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6sjr57crkxhh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a2572ed-7c03-46c2-98bb-89a19d5f777a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a2572ed-7c03-46c2-98bb-89a19d5f777a\",\n \"name\": \"3a2572ed-7c03-46c2-98bb-89a19d5f777a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:19:47.10992898Z\",\n \"endTime\": \"2026-10-02T03:38:53.263922354Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-9k9hg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-fjdfs: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-9k9hg: Creating: virtualmachine creating or updating\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-fjdfs: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000943f60\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-6sjr57crkxhh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a2572ed-7c03-46c2-98bb-89a19d5f777a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3a2572ed-7c03-46c2-98bb-89a19d5f777a\",\n \"name\": \"3a2572ed-7c03-46c2-98bb-89a19d5f777a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:19:47.10992898Z\",\n \"endTime\": \"2026-10-02T03:38:53.263922354Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-9k9hg: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-fjdfs: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-9k9hg: Creating: virtualmachine creating or updating\\nMachine d3m7o6a3u1p6z3i-np-1-vf5g4-fjdfs: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"72b0b59f816116f4fff569918df1af22c32e84bf1965b5228e7e015a6471ba06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"8750a839c279cdf29c1a82ccb6189d7fb26d4cb33d0d9e3708556a84f521bd7e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2rhqzm\" for cluster \"cluster-candidate-4-23-2rhqzm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90900\u003e: \n failed to create NodePool np-2rhqzm: failed waiting for nodepool=\"np-2rhqzm\" for cluster \"cluster-candidate-4-23-2rhqzm\" in resourcegroup=\"rg-candidate-4-23-hvftzsslsv5w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83438069-4881-4a7c-99c3-04167cb9898c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83438069-4881-4a7c-99c3-04167cb9898c\",\n \"name\": \"83438069-4881-4a7c-99c3-04167cb9898c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:40:49.053626001Z\",\n \"endTime\": \"2026-10-02T03:59:53.321378994Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-jdv5b: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-r825r: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-jdv5b: Creating: virtualmachine creating or updating\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-r825r: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2rhqzm\" for cluster \"cluster-candidate-4-23-2rhqzm\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90900\u003e: \n failed to create NodePool np-2rhqzm: failed waiting for nodepool=\"np-2rhqzm\" for cluster \"cluster-candidate-4-23-2rhqzm\" in resourcegroup=\"rg-candidate-4-23-hvftzsslsv5w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83438069-4881-4a7c-99c3-04167cb9898c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/83438069-4881-4a7c-99c3-04167cb9898c\",\n \"name\": \"83438069-4881-4a7c-99c3-04167cb9898c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:40:49.053626001Z\",\n \"endTime\": \"2026-10-02T03:59:53.321378994Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-jdv5b: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-r825r: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-jdv5b: Creating: virtualmachine creating or updating\\nMachine o2x4w0t0l5t3o4o-np-2rhqzm-ghd9l-r825r: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"74494e992ed4abcc34c99899efa3343e2c68f1c01682ea9b7400f4718bc995e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"9cbcd1fcc44b9dcfeac71a715da83e6c8d7bf9ceee862c10b305cfe44d73e05f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lfrdtx\" for cluster \"cluster-candidate-4-21-lfrdtx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a86ae0\u003e: \n failed to create NodePool np-lfrdtx: failed waiting for nodepool=\"np-lfrdtx\" for cluster \"cluster-candidate-4-21-lfrdtx\" in resourcegroup=\"rg-candidate-4-21-v7j8q5wv69rf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8487946-6d32-4b69-bbc8-793ffc91ce5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8487946-6d32-4b69-bbc8-793ffc91ce5f\",\n \"name\": \"a8487946-6d32-4b69-bbc8-793ffc91ce5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:40:32.227275462Z\",\n \"endTime\": \"2026-10-02T03:59:33.308103816Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-58v7d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-5gbq8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-58v7d: Creating: virtualmachine creating or updating\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-5gbq8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lfrdtx\" for cluster \"cluster-candidate-4-21-lfrdtx\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a86ae0\u003e: \n failed to create NodePool np-lfrdtx: failed waiting for nodepool=\"np-lfrdtx\" for cluster \"cluster-candidate-4-21-lfrdtx\" in resourcegroup=\"rg-candidate-4-21-v7j8q5wv69rf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8487946-6d32-4b69-bbc8-793ffc91ce5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a8487946-6d32-4b69-bbc8-793ffc91ce5f\",\n \"name\": \"a8487946-6d32-4b69-bbc8-793ffc91ce5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:40:32.227275462Z\",\n \"endTime\": \"2026-10-02T03:59:33.308103816Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-58v7d: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-5gbq8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-58v7d: Creating: virtualmachine creating or updating\\nMachine z7f1i8t4o9f5w4c-np-lfrdtx-fqngr-5gbq8: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"80cf20a9c9f509bba5e62e463711a1e1c06212f7805ee16146090e82e7566897","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"9011219d09cdf07f6d086e4ba313338a0c41037e23acedb902f10a2f46cf7b4c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-s4kd9s\" for cluster \"cluster-candidate-4-20-s4kd9s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0b4a0\u003e: \n failed to create NodePool np-s4kd9s: failed waiting for nodepool=\"np-s4kd9s\" for cluster \"cluster-candidate-4-20-s4kd9s\" in resourcegroup=\"rg-candidate-4-20-zlm28jv4jtcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\",\n \"name\": \"7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:38:55.251415601Z\",\n \"endTime\": \"2026-10-02T03:58:03.48228814Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-mn8wj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-st4n7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-mn8wj: Creating: virtualmachine creating or updating\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-st4n7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-s4kd9s\" for cluster \"cluster-candidate-4-20-s4kd9s\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b0b4a0\u003e: \n failed to create NodePool np-s4kd9s: failed waiting for nodepool=\"np-s4kd9s\" for cluster \"cluster-candidate-4-20-s4kd9s\" in resourcegroup=\"rg-candidate-4-20-zlm28jv4jtcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\",\n \"name\": \"7e5e7aaa-eb3f-4153-b0cf-40dab5c32152\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:38:55.251415601Z\",\n \"endTime\": \"2026-10-02T03:58:03.48228814Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-mn8wj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-st4n7: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-mn8wj: Creating: virtualmachine creating or updating\\nMachine j1u6v8i1i7q3s1b-np-s4kd9s-nmqgx-st4n7: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"994cb16080650a54048b98abc269c5ac388e88b8448457d0a7057e29e73b6ca5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"4bfb160db3532883f03c196af07faf62b876ad474cc0c49e5e501e7fd65537d8","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00064c1e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-gs752pln4txj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\",\n \"name\": \"9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:18:04.846590038Z\",\n \"endTime\": \"2026-10-02T03:37:13.231851998Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-5dbrz-khxf4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-5dbrz-khxf4: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00064c1e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-gs752pln4txj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\",\n \"name\": \"9d6e4c14-f5a6-4fd9-881e-2a42f87e3555\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:18:04.846590038Z\",\n \"endTime\": \"2026-10-02T03:37:13.231851998Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-5dbrz-khxf4: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-5dbrz-khxf4: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ad04048961b5f0b481e79c6e7a4ca829f00bffd14496dc3ff9939ace3fce0cd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"74c4a0991a9347ceeb2b3ab53adc95cedd09f120cf556321c9b74ae4d3616c90","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0011d4048\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rrrmbd9jmmdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d52358f-8309-4bc7-af9f-a5884f9307e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d52358f-8309-4bc7-af9f-a5884f9307e0\",\n \"name\": \"6d52358f-8309-4bc7-af9f-a5884f9307e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:16:58.467567062Z\",\n \"endTime\": \"2026-10-02T03:36:03.226032745Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-sqrfn-pjdbq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-sqrfn-vtbbr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-sqrfn-pjdbq: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-sqrfn-vtbbr: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc0011d4048\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rrrmbd9jmmdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d52358f-8309-4bc7-af9f-a5884f9307e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/6d52358f-8309-4bc7-af9f-a5884f9307e0\",\n \"name\": \"6d52358f-8309-4bc7-af9f-a5884f9307e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:16:58.467567062Z\",\n \"endTime\": \"2026-10-02T03:36:03.226032745Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-sqrfn-pjdbq: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-sqrfn-vtbbr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-sqrfn-pjdbq: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-sqrfn-vtbbr: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f3edaed51a0f30a57e00e2785d6ce36edebf24c4d9620d26d2389642b7c98f70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2105840312562749440","occurred_at":"2026-10-02T02:00:29Z","failed_at":"e2e","signature_id":"299db9852cb6ec237c0f1bb6b3917d59644cef430e83fc2d28e8943682b7e108","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2pqt8t\" for cluster \"cluster-candidate-5-0-2pqt8t\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e6c920\u003e: \n failed to create NodePool np-2pqt8t: failed waiting for nodepool=\"np-2pqt8t\" for cluster \"cluster-candidate-5-0-2pqt8t\" in resourcegroup=\"rg-candidate-5-0-5cmhl28njm2n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1671f2aa-76df-4172-89e0-297ac668f021\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1671f2aa-76df-4172-89e0-297ac668f021\",\n \"name\": \"1671f2aa-76df-4172-89e0-297ac668f021\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:42:21.546109728Z\",\n \"endTime\": \"2026-10-02T04:01:23.351194166Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-8bg9m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-v5mdw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-8bg9m: Creating: virtualmachine creating or updating\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-v5mdw: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2pqt8t\" for cluster \"cluster-candidate-5-0-2pqt8t\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e6c920\u003e: \n failed to create NodePool np-2pqt8t: failed waiting for nodepool=\"np-2pqt8t\" for cluster \"cluster-candidate-5-0-2pqt8t\" in resourcegroup=\"rg-candidate-5-0-5cmhl28njm2n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1671f2aa-76df-4172-89e0-297ac668f021\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1671f2aa-76df-4172-89e0-297ac668f021\",\n \"name\": \"1671f2aa-76df-4172-89e0-297ac668f021\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T03:42:21.546109728Z\",\n \"endTime\": \"2026-10-02T04:01:23.351194166Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-8bg9m: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-v5mdw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-8bg9m: Creating: virtualmachine creating or updating\\nMachine w7u4t7y3s9c5g5m-np-2pqt8t-xxc4f-v5mdw: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-02T00:53:40Z","started_at":"2026-10-02T00:53:40Z","completed_at":"2026-10-02T02:59:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-02T03:02:56.086682389Z","timing_metadata_checked_at":"2026-10-02T03:02:56.086682389Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-02T03:02:56.093669068Z","region_metadata_checked_at":"2026-10-02T03:02:56.093669068Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"3b16639bc16dbbe14f5810b05c126b6bd3984226ddf176e8d37bf1be9f2488f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"6d948816532f93e1d1f025d9ed2447096861bf76e34447e5173e22879b9214cf","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002434018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9gqhrxhspmxj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2689a9-2918-4b62-b602-77629a36d12f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2689a9-2918-4b62-b602-77629a36d12f\",\n \"name\": \"1e2689a9-2918-4b62-b602-77629a36d12f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:12:51.230795814Z\",\n \"endTime\": \"2026-10-02T01:31:55.658354454Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-zw5cd-7wmwm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-zw5cd-t5w4x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-zw5cd-7wmwm: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-zw5cd-t5w4x: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc002434018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-9gqhrxhspmxj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2689a9-2918-4b62-b602-77629a36d12f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/1e2689a9-2918-4b62-b602-77629a36d12f\",\n \"name\": \"1e2689a9-2918-4b62-b602-77629a36d12f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:12:51.230795814Z\",\n \"endTime\": \"2026-10-02T01:31:55.658354454Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-zw5cd-7wmwm: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine cilium-cluster-cilium-np-zw5cd-t5w4x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-zw5cd-7wmwm: Creating: virtualmachine creating or updating\\nMachine cilium-cluster-cilium-np-zw5cd-t5w4x: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6b4c95a67adaafe3b7d63c9fd89bab8a39011d0e1b1e5c8dba06b1307fe4076d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"9ab2b1eb14ea74c42017c9099045c52a9eb9611a341b4d1c5df66b6729fb582b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tgw4bw\" for cluster \"cluster-candidate-5-0-tgw4bw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013d7020\u003e: \n failed to create NodePool np-tgw4bw: failed waiting for nodepool=\"np-tgw4bw\" for cluster \"cluster-candidate-5-0-tgw4bw\" in resourcegroup=\"rg-candidate-5-0-c5dplrlznljs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4804927-bde6-4d56-b2a2-861fb62752d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4804927-bde6-4d56-b2a2-861fb62752d1\",\n \"name\": \"e4804927-bde6-4d56-b2a2-861fb62752d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:36:37.66754756Z\",\n \"endTime\": \"2026-10-02T01:55:45.209455595Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-h9x7v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-q4chj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-h9x7v: Creating: virtualmachine creating or updating\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-q4chj: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tgw4bw\" for cluster \"cluster-candidate-5-0-tgw4bw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013d7020\u003e: \n failed to create NodePool np-tgw4bw: failed waiting for nodepool=\"np-tgw4bw\" for cluster \"cluster-candidate-5-0-tgw4bw\" in resourcegroup=\"rg-candidate-5-0-c5dplrlznljs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4804927-bde6-4d56-b2a2-861fb62752d1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e4804927-bde6-4d56-b2a2-861fb62752d1\",\n \"name\": \"e4804927-bde6-4d56-b2a2-861fb62752d1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:36:37.66754756Z\",\n \"endTime\": \"2026-10-02T01:55:45.209455595Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-h9x7v: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-q4chj: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-h9x7v: Creating: virtualmachine creating or updating\\nMachine h0a6a6p2r2u4e6h-np-tgw4bw-s7jjh-q4chj: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6cfe946d5f677115c5c8031bc667f1dcea84285080095415791b7620da122103","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"a0aef2ff3133e203b8b6be69e707c3391a26d7478459a0d095a7d80d52b7f77e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101d200\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2lqkfn8tvwwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4098a97-bd38-463d-bee6-54c33fcd21ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4098a97-bd38-463d-bee6-54c33fcd21ea\",\n \"name\": \"f4098a97-bd38-463d-bee6-54c33fcd21ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:12:48.719133172Z\",\n \"endTime\": \"2026-10-02T01:31:55.837797735Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-kdjwf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-q7z6x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-kdjwf: Creating: virtualmachine creating or updating\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-q7z6x: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101d200\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2lqkfn8tvwwp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4098a97-bd38-463d-bee6-54c33fcd21ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/f4098a97-bd38-463d-bee6-54c33fcd21ea\",\n \"name\": \"f4098a97-bd38-463d-bee6-54c33fcd21ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:12:48.719133172Z\",\n \"endTime\": \"2026-10-02T01:31:55.837797735Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-kdjwf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-q7z6x: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-kdjwf: Creating: virtualmachine creating or updating\\nMachine b6g9d8r2h2w9x0t-np-1-qhfgz-q7z6x: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ecd9b8fd182cba134a98a738b7a020e73d790f531ab5de37cd2bbc7e510a7fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"2e01f42474260e70f5cc8017801f4c45f4df9dbc895992ce253206654bfb0587","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2dtcq8\" for cluster \"cluster-candidate-4-20-2dtcq8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018500e0\u003e: \n failed to create NodePool np-2dtcq8: failed waiting for nodepool=\"np-2dtcq8\" for cluster \"cluster-candidate-4-20-2dtcq8\" in resourcegroup=\"rg-candidate-4-20-wsgq7rfg4j9c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\",\n \"name\": \"4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:35:03.016964221Z\",\n \"endTime\": \"2026-10-02T01:54:05.336017308Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2dtcq8\" for cluster \"cluster-candidate-4-20-2dtcq8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0018500e0\u003e: \n failed to create NodePool np-2dtcq8: failed waiting for nodepool=\"np-2dtcq8\" for cluster \"cluster-candidate-4-20-2dtcq8\" in resourcegroup=\"rg-candidate-4-20-wsgq7rfg4j9c\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\",\n \"name\": \"4f8b062f-d3f0-4efa-9aa5-be9e1c5218c2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:35:03.016964221Z\",\n \"endTime\": \"2026-10-02T01:54:05.336017308Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ee29ba6ce642873f3ae4081c2b12ae165b5e71be7507b6512a426336aaa89af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"2c5ca9fee600aa40b6dd99a68079eed31e31c0041cca6c9bb81c4e2c28989095","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-z9vgpc\" for cluster \"cluster-candidate-4-23-z9vgpc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d433c0\u003e: \n failed to create NodePool np-z9vgpc: failed waiting for nodepool=\"np-z9vgpc\" for cluster \"cluster-candidate-4-23-z9vgpc\" in resourcegroup=\"rg-candidate-4-23-k2q8jrl7bhxd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5150081-901b-4236-a201-530c0a2b188a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5150081-901b-4236-a201-530c0a2b188a\",\n \"name\": \"a5150081-901b-4236-a201-530c0a2b188a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:35:33.904818612Z\",\n \"endTime\": \"2026-10-02T01:54:35.210883513Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-54zr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-tfmvd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-54zr9: Creating: virtualmachine creating or updating\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-tfmvd: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-z9vgpc\" for cluster \"cluster-candidate-4-23-z9vgpc\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d433c0\u003e: \n failed to create NodePool np-z9vgpc: failed waiting for nodepool=\"np-z9vgpc\" for cluster \"cluster-candidate-4-23-z9vgpc\" in resourcegroup=\"rg-candidate-4-23-k2q8jrl7bhxd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5150081-901b-4236-a201-530c0a2b188a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5150081-901b-4236-a201-530c0a2b188a\",\n \"name\": \"a5150081-901b-4236-a201-530c0a2b188a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:35:33.904818612Z\",\n \"endTime\": \"2026-10-02T01:54:35.210883513Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-54zr9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-tfmvd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-54zr9: Creating: virtualmachine creating or updating\\nMachine y0w7y2a8c7t4f6j-np-z9vgpc-4fpk5-tfmvd: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8884e88d6d84da2e7b9c0e73edf64fa7bf0b1182e9b1d3a8603c60c3f93abfec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"02f2d68ee929abf0d14dbf066cea47b45e31fa5a7861dd29f5ff1ae2dda13550","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d92dc0\u003e: \n failed to create NodePool np-ljll7c: failed waiting for nodepool=\"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\" in resourcegroup=\"rg-candidate-5-1-m96hr99msv6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\",\n \"name\": \"3472529a-993b-497a-a853-58314ee41fea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:39:28.8432233Z\",\n \"endTime\": \"2026-10-02T01:58:35.211672532Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002d92dc0\u003e: \n failed to create NodePool np-ljll7c: failed waiting for nodepool=\"np-ljll7c\" for cluster \"cluster-candidate-5-1-ljll7c\" in resourcegroup=\"rg-candidate-5-1-m96hr99msv6f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3472529a-993b-497a-a853-58314ee41fea\",\n \"name\": \"3472529a-993b-497a-a853-58314ee41fea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:39:28.8432233Z\",\n \"endTime\": \"2026-10-02T01:58:35.211672532Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9035c099feda9d5c8c3f53f6e9ceba973d7e11554bdb29525b4d3635fa47cd50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"0d8fad05bfc91d3bf4587ff521f51f71dec38c4bd6167c681739b8c407410634","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b95a80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-vp5rgzktgrqp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7950f938-55ee-4ce6-a5ca-480917c685bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7950f938-55ee-4ce6-a5ca-480917c685bf\",\n \"name\": \"7950f938-55ee-4ce6-a5ca-480917c685bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:15:07.199174306Z\",\n \"endTime\": \"2026-10-02T01:34:15.817819834Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-vqbnm-zdcgw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-vqbnm-zdcgw: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b95a80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-vp5rgzktgrqp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7950f938-55ee-4ce6-a5ca-480917c685bf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7950f938-55ee-4ce6-a5ca-480917c685bf\",\n \"name\": \"7950f938-55ee-4ce6-a5ca-480917c685bf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:15:07.199174306Z\",\n \"endTime\": \"2026-10-02T01:34:15.817819834Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-vqbnm-zdcgw: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-vqbnm-zdcgw: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c074fab4310135b3ac27c3fbc92a279b46d1962f358ad5fdfd4e9a262d6e651b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"556da369df5fa371fae5a268e2663151c01997d0b1338ed3e1e0b97061c14199","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pdsktt\" for cluster \"cluster-candidate-4-22-pdsktt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010d83a0\u003e: \n failed to create NodePool np-pdsktt: failed waiting for nodepool=\"np-pdsktt\" for cluster \"cluster-candidate-4-22-pdsktt\" in resourcegroup=\"rg-candidate-4-22-5hpdwnfcpnnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/987287c4-95cd-422d-be6d-282705a06618\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/987287c4-95cd-422d-be6d-282705a06618\",\n \"name\": \"987287c4-95cd-422d-be6d-282705a06618\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:35:11.708229062Z\",\n \"endTime\": \"2026-10-02T01:54:15.19502959Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-b9m75: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-vsxkb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-b9m75: Creating: virtualmachine creating or updating\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-vsxkb: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pdsktt\" for cluster \"cluster-candidate-4-22-pdsktt\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010d83a0\u003e: \n failed to create NodePool np-pdsktt: failed waiting for nodepool=\"np-pdsktt\" for cluster \"cluster-candidate-4-22-pdsktt\" in resourcegroup=\"rg-candidate-4-22-5hpdwnfcpnnf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/987287c4-95cd-422d-be6d-282705a06618\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/987287c4-95cd-422d-be6d-282705a06618\",\n \"name\": \"987287c4-95cd-422d-be6d-282705a06618\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:35:11.708229062Z\",\n \"endTime\": \"2026-10-02T01:54:15.19502959Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-b9m75: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-vsxkb: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-b9m75: Creating: virtualmachine creating or updating\\nMachine m9p9y7p7y8q8y0t-np-pdsktt-tcq2h-vsxkb: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"ccdac94ec54475c82fab76ae46c0faddb036dd10e1f61328d8867d84ed81d02a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"fcbce16b0ec7dd896386a692502ebe83bdec088d3b3acd3226882a8945ae4909","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001a9c060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-v2ffkzvzwhp9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/59c86be8-df72-4988-a51b-6b1b20b915f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/59c86be8-df72-4988-a51b-6b1b20b915f4\",\n \"name\": \"59c86be8-df72-4988-a51b-6b1b20b915f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:13:42.494621442Z\",\n \"endTime\": \"2026-10-02T01:32:45.646902114Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-69vhp-6nxhl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-69vhp-rch98: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-69vhp-6nxhl: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-69vhp-rch98: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001a9c060\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-v2ffkzvzwhp9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/59c86be8-df72-4988-a51b-6b1b20b915f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/59c86be8-df72-4988-a51b-6b1b20b915f4\",\n \"name\": \"59c86be8-df72-4988-a51b-6b1b20b915f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:13:42.494621442Z\",\n \"endTime\": \"2026-10-02T01:32:45.646902114Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-69vhp-6nxhl: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-69vhp-rch98: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-69vhp-6nxhl: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-69vhp-rch98: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"dbe220fd3c4eaa185ec9ed8726b0f7b0e126dc229bf9a0899c82093132d36286","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"5a5ab12ceb7567e88b54656617d4a4e3c761b84d7c19fe667549b41a60c4797b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tnjk89\" for cluster \"cluster-candidate-4-21-tnjk89\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a24560\u003e: \n failed to create NodePool np-tnjk89: failed waiting for nodepool=\"np-tnjk89\" for cluster \"cluster-candidate-4-21-tnjk89\" in resourcegroup=\"rg-candidate-4-21-hbwb4qtqm5m5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c80392bf-1395-461b-9b36-b3ca3dfa3732\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c80392bf-1395-461b-9b36-b3ca3dfa3732\",\n \"name\": \"c80392bf-1395-461b-9b36-b3ca3dfa3732\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:34:52.67307417Z\",\n \"endTime\": \"2026-10-02T01:53:55.817268557Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-mfsnf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-rmqgp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-mfsnf: Creating: virtualmachine creating or updating\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-rmqgp: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tnjk89\" for cluster \"cluster-candidate-4-21-tnjk89\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a24560\u003e: \n failed to create NodePool np-tnjk89: failed waiting for nodepool=\"np-tnjk89\" for cluster \"cluster-candidate-4-21-tnjk89\" in resourcegroup=\"rg-candidate-4-21-hbwb4qtqm5m5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c80392bf-1395-461b-9b36-b3ca3dfa3732\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c80392bf-1395-461b-9b36-b3ca3dfa3732\",\n \"name\": \"c80392bf-1395-461b-9b36-b3ca3dfa3732\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T01:34:52.67307417Z\",\n \"endTime\": \"2026-10-02T01:53:55.817268557Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-mfsnf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-rmqgp: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-mfsnf: Creating: virtualmachine creating or updating\\nMachine x5t3x3n0c0q1y5x-np-tnjk89-psj6d-rmqgp: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e8b6e5f980b89a3c447fa7c04b3863a137448f36c1a35ce9db2eb93a64f16e36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2105823495052070912","occurred_at":"2026-10-02T00:53:40Z","failed_at":"e2e","signature_id":"4dcb3e82c312b27beb2846bcb4e3feac7ef1c6c538ba9c325c9e8ba1cd984500","test_name":"Customer should be able to delete a nodepool","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000eaa180\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ks2bs7m442f9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3917dbfd-2f9f-4a35-8346-d9c0b133e313\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3917dbfd-2f9f-4a35-8346-d9c0b133e313\",\n \"name\": \"3917dbfd-2f9f-4a35-8346-d9c0b133e313\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T02:04:51.953995015Z\",\n \"endTime\": \"2026-10-02T02:23:58.43798257Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-85c4g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-g4qwf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-85c4g: Creating: virtualmachine creating or updating\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-g4qwf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000eaa180\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-ks2bs7m442f9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3917dbfd-2f9f-4a35-8346-d9c0b133e313\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/3917dbfd-2f9f-4a35-8346-d9c0b133e313\",\n \"name\": \"3917dbfd-2f9f-4a35-8346-d9c0b133e313\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-10-02T02:04:51.953995015Z\",\n \"endTime\": \"2026-10-02T02:23:58.43798257Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-85c4g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-g4qwf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-85c4g: Creating: virtualmachine creating or updating\\nMachine w5t1z3a1d2p4k6h-np-deletable-l2j9j-g4qwf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5rf62rf4p9wv\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["37f828a522b8f965e935d01a24c030fdfedd0a9019cc49b71eb9f52be0c9df86","839b3fb6e0d5a44ab3b02f9de591e8eb36fc8b7a3a10fc47bff4fa4fe479f219","98930009f81aef2f1925b35a9524eccf816ff61caee985eb93016341ef1e8567","fcc0b2218a239300afa0d89b62b501f08536fb474cd86eb6594826af9da09c79"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}