{"meta":{"date":"2026-09-28","timezone":"UTC","generated_at":"2026-10-31T04:54:04Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":13,"failed_runs":7,"runs_with_occurrences":7,"runs_with_failure_pattern_matches":7,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":6,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T23:22:35Z","started_at":"2026-09-28T23:22:35Z","completed_at":"2026-09-29T01:47:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:55:31.127655818Z","timing_metadata_checked_at":"2026-09-29T01:55:31.127655818Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:55:31.130521909Z","region_metadata_checked_at":"2026-09-29T01:55:31.130521909Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"48b59f3875e9ba2e31c5adcb9ded0486e232ebf2e75ecb6769908260d7c9b23c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","occurred_at":"2026-09-28T23:22:35Z","failed_at":"e2e","signature_id":"29bde2d7b8d5a41edc15f4349385c2d0099553bdf943c64bb8c46da6c996a3e1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00087e120\u003e: \n    failed to create NodePool np-rcls6m: failed waiting for nodepool=\"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\" in resourcegroup=\"rg-candidate-4-21-92w77dwp5rg8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"name\": \"a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:31.628142216Z\",\n      \"endTime\": \"2026-09-29T00:35:38.838060326Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00087e120\u003e: \n    failed to create NodePool np-rcls6m: failed waiting for nodepool=\"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\" in resourcegroup=\"rg-candidate-4-21-92w77dwp5rg8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"name\": \"a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:31.628142216Z\",\n      \"endTime\": \"2026-09-29T00:35:38.838060326Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dd115327b69c3b100b2ab8f12a8a65af701b609a8befa9d6488273ed31b7c8ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","occurred_at":"2026-09-28T23:22:35Z","failed_at":"e2e","signature_id":"057d78f20fc782ce9997cb17d11d93f7004a1d7a6ecbd74b197c32efd5c0100d","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b43f40\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b43f40\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T23:11:16Z","started_at":"2026-09-28T23:11:16Z","completed_at":"2026-09-29T01:51:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T02:06:08.08290612Z","timing_metadata_checked_at":"2026-09-29T02:06:08.08290612Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T02:06:08.0887953Z","region_metadata_checked_at":"2026-09-29T02:06:08.0887953Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"0dbd96b95b7f6de6d927e9f8622c4c6d6178fc538445f547e0865b8456950faa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"9b958b779ca895c4c49a73903b6b72b8677fe49ebfe23b8787341a9e83c6884a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010410e0\u003e: \n    failed to create NodePool np-5m4wlb: failed waiting for nodepool=\"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\" in resourcegroup=\"rg-candidate-4-23-wwwsqs8cghpq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"name\": \"91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:18:19.018330415Z\",\n      \"endTime\": \"2026-09-29T00:37:28.829514453Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: Creating: virtualmachine creating or updating\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010410e0\u003e: \n    failed to create NodePool np-5m4wlb: failed waiting for nodepool=\"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\" in resourcegroup=\"rg-candidate-4-23-wwwsqs8cghpq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"name\": \"91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:18:19.018330415Z\",\n      \"endTime\": \"2026-09-29T00:37:28.829514453Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: Creating: virtualmachine creating or updating\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"46ec4dc1ee77f2b04feaf8a4e870acc27d6af0c3513836a206a1d5f95dff9612","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"0950bf1fa4b422939bc22e6eec036cdf20bb4658d9c093acba892685503bea17","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bad000\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-64htmh\" in resourcegroup=\"rg-np-noedge-64htmh-c4j9qdqfrq4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"name\": \"3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:00:25.564607872Z\",\n      \"endTime\": \"2026-09-29T01:19:27.08336887Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: Creating: virtualmachine creating or updating\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bad000\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-64htmh\" in resourcegroup=\"rg-np-noedge-64htmh-c4j9qdqfrq4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"name\": \"3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:00:25.564607872Z\",\n      \"endTime\": \"2026-09-29T01:19:27.08336887Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: Creating: virtualmachine creating or updating\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6681890728c7562f98f797c9fff59b9157e7a6a75b3b89f7fde64576735f17ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"e4912b2f6d7e7563f8767c16448cd57abe8e92db7831d87c4f122e3d8fc2c77d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-p28f7h\" for cluster \"cluster-candidate-4-22-p28f7h\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00068b3c0\u003e: \n    failed to create NodePool np-p28f7h: failed waiting for nodepool=\"np-p28f7h\" for cluster \"cluster-candidate-4-22-p28f7h\" in resourcegroup=\"rg-candidate-4-22-mrpj7hdjdh7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/45638889-70a7-4fdc-8d99-520c3c98a19b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/45638889-70a7-4fdc-8d99-520c3c98a19b\",\n      \"name\": \"45638889-70a7-4fdc-8d99-520c3c98a19b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:17:24.13464273Z\",\n      \"endTime\": \"2026-09-29T00:36:28.832441888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a9f5u6l9p6p9x0x-np-p28f7h-pwbqm-k6srx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a9f5u6l9p6p9x0x-np-p28f7h-pwbqm-k6srx: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-p28f7h\" for cluster \"cluster-candidate-4-22-p28f7h\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00068b3c0\u003e: \n    failed to create NodePool np-p28f7h: failed waiting for nodepool=\"np-p28f7h\" for cluster \"cluster-candidate-4-22-p28f7h\" in resourcegroup=\"rg-candidate-4-22-mrpj7hdjdh7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/45638889-70a7-4fdc-8d99-520c3c98a19b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/45638889-70a7-4fdc-8d99-520c3c98a19b\",\n      \"name\": \"45638889-70a7-4fdc-8d99-520c3c98a19b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:17:24.13464273Z\",\n      \"endTime\": \"2026-09-29T00:36:28.832441888Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a9f5u6l9p6p9x0x-np-p28f7h-pwbqm-k6srx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a9f5u6l9p6p9x0x-np-p28f7h-pwbqm-k6srx: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8dc3aee5dc9358deaaabc1cedebb19a6707e4f408d61dc9277f1c417f5cfbae7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"f07ab4b774d34743455c8d173177f39c64d2818384324244b685d13e7ea7d3cc","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0007275a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdcdkl9rvn5l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\",\n      \"name\": \"3a70e081-8150-479f-812b-31eb02522612\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:01:02.974562787Z\",\n      \"endTime\": \"2026-09-29T01:20:06.931508754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0007275a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdcdkl9rvn5l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\",\n      \"name\": \"3a70e081-8150-479f-812b-31eb02522612\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:01:02.974562787Z\",\n      \"endTime\": \"2026-09-29T01:20:06.931508754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management"}},{"row_id":"a44157a6759cdfba7890fa3a3cdd707a5092f989f44725faa6698883b24cfd89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"053123c7443d1054083dcab559d2546849eae900c250f58b40fbf47904d636dc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7f080\u003e: \n    failed to create NodePool np-bhxv8b: failed waiting for nodepool=\"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\" in resourcegroup=\"rg-candidate-4-21-mrwxnrjtcxks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"name\": \"b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:53.990233971Z\",\n      \"endTime\": \"2026-09-29T00:35:58.987434024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7f080\u003e: \n    failed to create NodePool np-bhxv8b: failed waiting for nodepool=\"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\" in resourcegroup=\"rg-candidate-4-21-mrwxnrjtcxks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"name\": \"b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:53.990233971Z\",\n      \"endTime\": \"2026-09-29T00:35:58.987434024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T23:00:26Z","started_at":"2026-09-28T23:00:26Z","completed_at":"2026-09-29T00:50:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:04:03.677054674Z","timing_metadata_checked_at":"2026-09-29T01:04:03.677054674Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:04:03.679909965Z","region_metadata_checked_at":"2026-09-29T01:04:03.679909965Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"1b66d8f3a5c143c96942e5ece4d9e241960b248975191a56cffb98a298ebfc2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"e7f9b8ac38554f62beb05357bcb84d1c79a8ef404e325310c520ae6e5285e61d","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f35488\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f35488\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"25ac2f0b85e97a163367bb2843c94b3527c743e45785d1a5f23675cca738d2e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"7732b8465eb0b310323c46363238564ae3708ad1dd66f83e67d67744e2112be2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00179cc60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-v4rx8l2tw9wq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00179cc60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-v4rx8l2tw9wq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"46ff0b3b33bfe312808502d2dd00d17cb9aff1476f477652ded7346401818c6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"9aafea9b5d579d12248bc117bbaa084f2543cc7e5eb735c49007499bd1c1c528","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000aa43c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qgqdfprzx89s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000aa43c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qgqdfprzx89s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5abe172d06b02fda37f1134af66e71d315c5b76f67bf0b45846b465a48d3f971","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"56f51dbe4172c7698c7e2ac9b44cc9309db978fcf5349649586b91b53538f803","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048440\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-k6t82b\" resourcegroup=\"rg-np-dg-minor-k6t82b-4f2hjngd28k4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-k6t82b-4f2hjngd28k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-k6t82b/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048440\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-k6t82b\" resourcegroup=\"rg-np-dg-minor-k6t82b-4f2hjngd28k4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-k6t82b-4f2hjngd28k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-k6t82b/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6219e83d01cce0fe664189ba73a5771ea599b137e585f4fc68fcc60f211a082d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"8bfd58413915673e86f8668f73c4a1593d6b25ac14d3df8cca7f16fa47901f61","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24340\u003e: \n    failed to create NodePool np-5cw4w9: failed starting nodepool creation \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\" in resourcegroup=\"rg-nightly-4-20-b7gtl2svm55n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-b7gtl2svm55n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-5cw4w9/nodePools/np-5cw4w9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24340\u003e: \n    failed to create NodePool np-5cw4w9: failed starting nodepool creation \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\" in resourcegroup=\"rg-nightly-4-20-b7gtl2svm55n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-b7gtl2svm55n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-5cw4w9/nodePools/np-5cw4w9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6afb64dd831e95d14afa16946a4d93d3e2fa42baf141d0f4ddd715171ad8fb94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"f675dd7539b7a7c84d8073b82c8ff50aab68a3afafa2637412a75cd680bb8823","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0024d21b0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gd648czgpsjb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0024d21b0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gd648czgpsjb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"84ab21d5da097a4526804ad15e8f9cffda43e128c39580ec8184c1e7c3d7e8a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"61da179f0b7befc0f6ef4e2c530e9e4e588275b255eb735499eff0b6124e8ce5","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001055c80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tf5tm29lbrgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"name\": \"51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:10:33.615095623Z\",\n      \"endTime\": \"2026-09-28T23:29:40.114080242Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001055c80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tf5tm29lbrgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"name\": \"51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:10:33.615095623Z\",\n      \"endTime\": \"2026-09-28T23:29:40.114080242Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management"}},{"row_id":"850fea80834522a2f88a1d806416b86cdd1bb884b6af42102c651dc6a93818c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"5e02d079f2bb2b4baf359fdcea15f52be4ed619abb6034bc78cd714b27a3f342","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ade640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-t7jl46\" in resourcegroup=\"rg-np-version-upgrade-t7jl46-65khsnvbc4zl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t7jl46-65khsnvbc4zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t7jl46/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ade640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-t7jl46\" in resourcegroup=\"rg-np-version-upgrade-t7jl46-65khsnvbc4zl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t7jl46-65khsnvbc4zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t7jl46/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"dbd368e1e7ed04d70004c589fa81a2bd37ea295dd551b5b55e32d742d8811bb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"8de98f59efa15f72a396ca1bd7e2dff75eb6bfdd7ab655e62dff7e62b3c1152d","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00130ef60\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7g7qln9l2np5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"name\": \"ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:42:15.584467675Z\",\n      \"endTime\": \"2026-09-29T00:01:22.906593334Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00130ef60\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7g7qln9l2np5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"name\": \"ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:42:15.584467675Z\",\n      \"endTime\": \"2026-09-29T00:01:22.906593334Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dc30b5999fa75fa9faa92b3403f8581713a3bb1da1f30dd90649307bc3651c16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"8bea3b31627e5d847815834266e4adb45e346c49dee4938fc8669a5a0d148891","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0030a64a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kx45mg6gnp52\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\",\n      \"name\": \"7aa70862-e406-44df-915d-27100318e68b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:56:08.330966898Z\",\n      \"endTime\": \"2026-09-29T00:15:12.920582798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0030a64a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kx45mg6gnp52\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\",\n      \"name\": \"7aa70862-e406-44df-915d-27100318e68b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:56:08.330966898Z\",\n      \"endTime\": \"2026-09-29T00:15:12.920582798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management"}},{"row_id":"e320dea94e37550b9886781be4c378f5098a37633e42caa8dc6ecaf70e575e2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"c36a633bcd07dffb48200bace887b57ae6fe20eba4fb59a0da31d14814ce0b59","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000831500\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-zvgs4q\" in resourcegroup=\"rg-np-skip-minor-zvgs4q-797p7px4gkts\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-zvgs4q-797p7px4gkts/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-zvgs4q/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000831500\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-zvgs4q\" in resourcegroup=\"rg-np-skip-minor-zvgs4q-797p7px4gkts\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-zvgs4q-797p7px4gkts/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-zvgs4q/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f155dc3cecff24d24c50582a0a3b3936d18432ea8ccdbfd78cb0a962cbe22de8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"d33cdf557cf6b0a812334a9c133b09038e1b46fff4e53cc99d59b39fb3a25778","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-q9zwpnrqm4zb/cluster-nightly-5-1-sq64f4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110cda0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-sq64f4: failed waiting for cluster=\"cluster-nightly-5-1-sq64f4\" in resourcegroup=\"rg-nightly-5-1-q9zwpnrqm4zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\",\n      \"name\": \"25287b3f-b870-4e18-a51f-514432892aec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:31:45.913312Z\",\n      \"endTime\": \"2026-09-29T00:00:52.759469958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-q9zwpnrqm4zb/cluster-nightly-5-1-sq64f4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110cda0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-sq64f4: failed waiting for cluster=\"cluster-nightly-5-1-sq64f4\" in resourcegroup=\"rg-nightly-5-1-q9zwpnrqm4zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\",\n      \"name\": \"25287b3f-b870-4e18-a51f-514432892aec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:31:45.913312Z\",\n      \"endTime\": \"2026-09-29T00:00:52.759469958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f16c375021ac3e568d45f076ae016cf4ba3ca1316b15e9eb5c118b9af2ad0e02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"6cce0979bce816f94e1a76410fe9b0dfddfaeb9167c8d068f6c6c58b8957a7db","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a51a10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mnwb4zfspsdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a51a10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mnwb4zfspsdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"fbc322678c5e06f92fab954b06af0fe8f945304394da5981fae2e46f7a32fea1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"dd38e1bb9f1353d3c3ede4dc31e69fe80aa02069128581607c7d92391c522e32","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008a5e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-282f9ccvqths/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0008a5e00\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-282f9ccvqths/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104670218818162688","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T20:30:57Z","started_at":"2026-09-28T20:30:57Z","completed_at":"2026-09-28T23:04:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T23:18:08.360052763Z","timing_metadata_checked_at":"2026-09-28T23:18:08.360052763Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T23:18:08.362785354Z","region_metadata_checked_at":"2026-09-28T23:18:08.362785354Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104669292057333760","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T20:27:16Z","started_at":"2026-09-28T20:27:16Z","completed_at":"2026-09-28T23:05:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T23:18:08.334512944Z","timing_metadata_checked_at":"2026-09-28T23:18:08.334512944Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T23:18:08.337946533Z","region_metadata_checked_at":"2026-09-28T23:18:08.337946533Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"54a7f49f5ea2b7d79828ae614a6830162841b03408761eb3510e192edaa0a2d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104669292057333760","occurred_at":"2026-09-28T20:27:16Z","failed_at":"e2e","signature_id":"d239f5d1b83b70a7a970eaac57a5e3865875c1f179ca44cad09e2f730112d858","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00171b4a0\u003e: \n    failed to create NodePool np-nmsq2l: failed waiting for nodepool=\"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\" in resourcegroup=\"rg-candidate-4-21-lf4lgs4pzp7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\",\n      \"name\": \"8303805f-b000-457e-b724-d4df7eac6975\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:30:59.759776246Z\",\n      \"endTime\": \"2026-09-28T21:50:00.145843425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00171b4a0\u003e: \n    failed to create NodePool np-nmsq2l: failed waiting for nodepool=\"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\" in resourcegroup=\"rg-candidate-4-21-lf4lgs4pzp7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\",\n      \"name\": \"8303805f-b000-457e-b724-d4df7eac6975\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:30:59.759776246Z\",\n      \"endTime\": \"2026-09-28T21:50:00.145843425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T20:21:29Z","started_at":"2026-09-28T20:21:29Z","completed_at":"2026-09-28T22:58:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T23:07:16.855751151Z","timing_metadata_checked_at":"2026-09-28T23:07:16.855751151Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T23:07:16.858621241Z","region_metadata_checked_at":"2026-09-28T23:07:16.858621241Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"23efdf86e42e91829cc1e82432f2e1321fc28ffbbb5b7751faa235c52a1337bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","occurred_at":"2026-09-28T20:21:29Z","failed_at":"e2e","signature_id":"b06391e0cd724ab0d438d9c0420d007165f3fe823ffd52497e5d58364ca57ca5","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001139d00\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001139d00\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred"}},{"row_id":"7b82c2406993609ab9d7ac74c1d07cde77674dc272b657170b3363d34f0e2332","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","occurred_at":"2026-09-28T20:21:29Z","failed_at":"e2e","signature_id":"0808cac9ab1e6eab75a259dd85739aff3ad9bf80be0b493ce8ad127f37249569","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8vn5sb\" for cluster \"cluster-candidate-4-20-8vn5sb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00081f3e0\u003e: \n    failed to create NodePool np-8vn5sb: failed waiting for nodepool=\"np-8vn5sb\" for cluster \"cluster-candidate-4-20-8vn5sb\" in resourcegroup=\"rg-candidate-4-20-2kxj4bcd64pd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5b06b703-2ea9-4252-ab56-c1c6946ec407\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5b06b703-2ea9-4252-ab56-c1c6946ec407\",\n      \"name\": \"5b06b703-2ea9-4252-ab56-c1c6946ec407\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:21:03.360782437Z\",\n      \"endTime\": \"2026-09-28T21:40:07.046700763Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-2kxj4bcd64pd-candidate-8vn5sb--managed/providers/Microsoft.Compute/virtualMachines/b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-2kxj4bcd64pd-candidate-8vn5sb--managed/providers/Microsoft.Compute/virtualMachines/b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8vn5sb\" for cluster \"cluster-candidate-4-20-8vn5sb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00081f3e0\u003e: \n    failed to create NodePool np-8vn5sb: failed waiting for nodepool=\"np-8vn5sb\" for cluster \"cluster-candidate-4-20-8vn5sb\" in resourcegroup=\"rg-candidate-4-20-2kxj4bcd64pd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5b06b703-2ea9-4252-ab56-c1c6946ec407\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5b06b703-2ea9-4252-ab56-c1c6946ec407\",\n      \"name\": \"5b06b703-2ea9-4252-ab56-c1c6946ec407\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:21:03.360782437Z\",\n      \"endTime\": \"2026-09-28T21:40:07.046700763Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-2kxj4bcd64pd-candidate-8vn5sb--managed/providers/Microsoft.Compute/virtualMachines/b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-2kxj4bcd64pd-candidate-8vn5sb--managed/providers/Microsoft.Compute/virtualMachines/b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104621627147816960","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T17:17:52Z","started_at":"2026-09-28T17:17:52Z","completed_at":"2026-09-28T19:29:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T19:36:05.325980711Z","timing_metadata_checked_at":"2026-09-28T19:36:05.325980711Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T19:36:05.330897695Z","region_metadata_checked_at":"2026-09-28T19:36:05.330897695Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104576811751968768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T14:19:44Z","started_at":"2026-09-28T14:19:44Z","completed_at":"2026-09-28T16:44:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:58:44.666691228Z","timing_metadata_checked_at":"2026-09-28T16:58:44.666691228Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:58:44.66937642Z","region_metadata_checked_at":"2026-09-28T16:58:44.66937642Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2f5e6c6ad65e2a0121ff114afd2b4a75e84b4bd52204ea4153b9ddf64ae49087","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104576811751968768","occurred_at":"2026-09-28T14:19:44Z","failed_at":"e2e","signature_id":"31429d7beca55879e992a9e74912d2072a12718cdce9ba4d4ebbfb9d70101e40","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0006d65e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1ed0a0-a90c-4672-a753-87b575007435\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1ed0a0-a90c-4672-a753-87b575007435\",\n      \"name\": \"cf1ed0a0-a90c-4672-a753-87b575007435\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T14:48:00.539019012Z\",\n      \"endTime\": \"2026-09-28T15:07:01.911525796Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-rhtj7-kpztn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tdccrn4dc9dh--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-rhtj7-kpztn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-rhtj7-kpztn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tdccrn4dc9dh--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-rhtj7-kpztn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0006d65e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1ed0a0-a90c-4672-a753-87b575007435\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1ed0a0-a90c-4672-a753-87b575007435\",\n      \"name\": \"cf1ed0a0-a90c-4672-a753-87b575007435\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T14:48:00.539019012Z\",\n      \"endTime\": \"2026-09-28T15:07:01.911525796Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-rhtj7-kpztn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tdccrn4dc9dh--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-rhtj7-kpztn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-rhtj7-kpztn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tdccrn4dc9dh--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-rhtj7-kpztn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104574026838970368","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T14:08:43Z","started_at":"2026-09-28T14:08:43Z","completed_at":"2026-09-28T16:46:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:58:44.53717422Z","timing_metadata_checked_at":"2026-09-28T16:58:44.53717422Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:58:44.541824106Z","region_metadata_checked_at":"2026-09-28T16:58:44.541824106Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104573697271533568","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T14:07:25Z","started_at":"2026-09-28T14:07:25Z","completed_at":"2026-09-28T16:45:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:58:44.411694599Z","timing_metadata_checked_at":"2026-09-28T16:58:44.411694599Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:58:44.414248492Z","region_metadata_checked_at":"2026-09-28T16:58:44.414248492Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104573041903144960","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T14:04:48Z","started_at":"2026-09-28T14:04:48Z","completed_at":"2026-09-28T16:47:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:58:44.280748295Z","timing_metadata_checked_at":"2026-09-28T16:58:44.280748295Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:58:44.284454484Z","region_metadata_checked_at":"2026-09-28T16:58:44.284454484Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104520260127297536","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T10:35:04Z","started_at":"2026-09-28T10:35:04Z","completed_at":"2026-09-28T13:08:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T13:18:05.760444839Z","timing_metadata_checked_at":"2026-09-28T13:18:05.760444839Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T13:18:05.763564328Z","region_metadata_checked_at":"2026-09-28T13:18:05.763564328Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104390744759341056","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T02:00:25Z","started_at":"2026-09-28T02:00:25Z","completed_at":"2026-09-28T04:24:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T04:41:57.432237685Z","timing_metadata_checked_at":"2026-09-28T04:41:57.432237685Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T04:41:57.435441175Z","region_metadata_checked_at":"2026-09-28T04:41:57.435441175Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"88a70da9bc66a48eda9a9fe148a00700d48888bc71c8a672c3d835e48c7f12f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104390744759341056","occurred_at":"2026-09-28T02:00:25Z","failed_at":"e2e","signature_id":"08f63884b8084a2606a1a5268b6fb65c1cc3e789a692cbd59f3ad65ea31fbc51","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4z25qr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7lft27tlvnbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0009fab20\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4z25qr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7lft27tlvnbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0009fab20\u003e: \n    storage account log verification failed\n    ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}