{"meta":{"start_date":"2026-09-28","end_date":"2026-09-28","start_at":"2026-09-28T00:00:00Z","end_at":"2026-09-29T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T08:08:57Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":13,"failed_runs":7,"raw_occurrences":26,"matched_occurrences":26,"runs_affected":7},"rows":[{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":5,"after_last_push_count":58,"trend_counts":[5,4,5,5,5,0,5],"trend_range":"2026-09-22..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a51a10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mnwb4zfspsdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a51a10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mnwb4zfspsdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00179cc60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-v4rx8l2tw9wq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00179cc60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-v4rx8l2tw9wq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000aa43c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qgqdfprzx89s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000aa43c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qgqdfprzx89s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"6cce0979bce816f94e1a76410fe9b0dfddfaeb9167c8d068f6c6c58b8957a7db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"7732b8465eb0b310323c46363238564ae3708ad1dd66f83e67d67744e2112be2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"9aafea9b5d579d12248bc117bbaa084f2543cc7e5eb735c49007499bd1c1c528","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"dd38e1bb9f1353d3c3ede4dc31e69fe80aa02069128581607c7d92391c522e32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"f675dd7539b7a7c84d8073b82c8ff50aab68a3afafa2637412a75cd680bb8823","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":30.76923076923077,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-09-22..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00087e120\u003e: \n    failed to create NodePool np-rcls6m: failed waiting for nodepool=\"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\" in resourcegroup=\"rg-candidate-4-21-92w77dwp5rg8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"name\": \"a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:31.628142216Z\",\n      \"endTime\": \"2026-09-29T00:35:38.838060326Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00087e120\u003e: \n    failed to create NodePool np-rcls6m: failed waiting for nodepool=\"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\" in resourcegroup=\"rg-candidate-4-21-92w77dwp5rg8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"name\": \"a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:31.628142216Z\",\n      \"endTime\": \"2026-09-29T00:35:38.838060326Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00130ef60\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7g7qln9l2np5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"name\": \"ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:42:15.584467675Z\",\n      \"endTime\": \"2026-09-29T00:01:22.906593334Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00130ef60\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7g7qln9l2np5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"name\": \"ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:42:15.584467675Z\",\n      \"endTime\": \"2026-09-29T00:01:22.906593334Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00171b4a0\u003e: \n    failed to create NodePool np-nmsq2l: failed waiting for nodepool=\"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\" in resourcegroup=\"rg-candidate-4-21-lf4lgs4pzp7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\",\n      \"name\": \"8303805f-b000-457e-b724-d4df7eac6975\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:30:59.759776246Z\",\n      \"endTime\": \"2026-09-28T21:50:00.145843425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00171b4a0\u003e: \n    failed to create NodePool np-nmsq2l: failed waiting for nodepool=\"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\" in resourcegroup=\"rg-candidate-4-21-lf4lgs4pzp7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\",\n      \"name\": \"8303805f-b000-457e-b724-d4df7eac6975\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T21:30:59.759776246Z\",\n      \"endTime\": \"2026-09-28T21:50:00.145843425Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","occurred_at":"2026-09-28T23:22:35Z","signature_id":"29bde2d7b8d5a41edc15f4349385c2d0099553bdf943c64bb8c46da6c996a3e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"8de98f59efa15f72a396ca1bd7e2dff75eb6bfdd7ab655e62dff7e62b3c1152d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104669292057333760","occurred_at":"2026-09-28T20:27:16Z","signature_id":"d239f5d1b83b70a7a970eaac57a5e3865875c1f179ca44cad09e2f730112d858","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","occurred_at":"2026-09-28T20:21:29Z","signature_id":"0808cac9ab1e6eab75a259dd85739aff3ad9bf80be0b493ce8ad127f37249569","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":4,"runs_affected":3,"failed_runs":3,"run_impact_percent":23.076923076923077,"anchor_occurrences":4,"after_last_push_count":7,"trend_counts":[0,0,1,1,0,0,4],"trend_range":"2026-09-22..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-25T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0007275a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdcdkl9rvn5l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\",\n      \"name\": \"3a70e081-8150-479f-812b-31eb02522612\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:01:02.974562787Z\",\n      \"endTime\": \"2026-09-29T01:20:06.931508754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0007275a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdcdkl9rvn5l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\",\n      \"name\": \"3a70e081-8150-479f-812b-31eb02522612\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:01:02.974562787Z\",\n      \"endTime\": \"2026-09-29T01:20:06.931508754Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001055c80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tf5tm29lbrgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"name\": \"51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:10:33.615095623Z\",\n      \"endTime\": \"2026-09-28T23:29:40.114080242Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001055c80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tf5tm29lbrgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"name\": \"51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:10:33.615095623Z\",\n      \"endTime\": \"2026-09-28T23:29:40.114080242Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0030a64a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kx45mg6gnp52\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\",\n      \"name\": \"7aa70862-e406-44df-915d-27100318e68b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:56:08.330966898Z\",\n      \"endTime\": \"2026-09-29T00:15:12.920582798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0030a64a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kx45mg6gnp52\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\",\n      \"name\": \"7aa70862-e406-44df-915d-27100318e68b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:56:08.330966898Z\",\n      \"endTime\": \"2026-09-29T00:15:12.920582798Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"f07ab4b774d34743455c8d173177f39c64d2818384324244b685d13e7ea7d3cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"61da179f0b7befc0f6ef4e2c530e9e4e588275b255eb735499eff0b6124e8ce5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"8bea3b31627e5d847815834266e4adb45e346c49dee4938fc8669a5a0d148891","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104576811751968768","occurred_at":"2026-09-28T14:19:44Z","signature_id":"31429d7beca55879e992a9e74912d2072a12718cdce9ba4d4ebbfb9d70101e40","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-09-22..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7f080\u003e: \n    failed to create NodePool np-bhxv8b: failed waiting for nodepool=\"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\" in resourcegroup=\"rg-candidate-4-21-mrwxnrjtcxks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"name\": \"b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:53.990233971Z\",\n      \"endTime\": \"2026-09-29T00:35:58.987434024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7f080\u003e: \n    failed to create NodePool np-bhxv8b: failed waiting for nodepool=\"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\" in resourcegroup=\"rg-candidate-4-21-mrwxnrjtcxks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"name\": \"b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:16:53.990233971Z\",\n      \"endTime\": \"2026-09-29T00:35:58.987434024Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bad000\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-64htmh\" in resourcegroup=\"rg-np-noedge-64htmh-c4j9qdqfrq4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"name\": \"3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:00:25.564607872Z\",\n      \"endTime\": \"2026-09-29T01:19:27.08336887Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: Creating: virtualmachine creating or updating\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000bad000\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-64htmh\" in resourcegroup=\"rg-np-noedge-64htmh-c4j9qdqfrq4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"name\": \"3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T01:00:25.564607872Z\",\n      \"endTime\": \"2026-09-29T01:19:27.08336887Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: Creating: virtualmachine creating or updating\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010410e0\u003e: \n    failed to create NodePool np-5m4wlb: failed waiting for nodepool=\"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\" in resourcegroup=\"rg-candidate-4-23-wwwsqs8cghpq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"name\": \"91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:18:19.018330415Z\",\n      \"endTime\": \"2026-09-29T00:37:28.829514453Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: Creating: virtualmachine creating or updating\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010410e0\u003e: \n    failed to create NodePool np-5m4wlb: failed waiting for nodepool=\"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\" in resourcegroup=\"rg-candidate-4-23-wwwsqs8cghpq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"name\": \"91d9873c-e44e-4e44-ba41-ade87f500052\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-29T00:18:19.018330415Z\",\n      \"endTime\": \"2026-09-29T00:37:28.829514453Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: Creating: virtualmachine creating or updating\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"053123c7443d1054083dcab559d2546849eae900c250f58b40fbf47904d636dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"0950bf1fa4b422939bc22e6eec036cdf20bb4658d9c093acba892685503bea17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"9b958b779ca895c4c49a73903b6b72b8677fe49ebfe23b8787341a9e83c6884a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","signature_id":"e4912b2f6d7e7563f8767c16448cd57abe8e92db7831d87c4f122e3d8fc2c77d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":4,"after_last_push_count":46,"trend_counts":[4,4,4,4,2,4,4],"trend_range":"2026-09-22..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048440\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-k6t82b\" resourcegroup=\"rg-np-dg-minor-k6t82b-4f2hjngd28k4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-k6t82b-4f2hjngd28k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-k6t82b/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001048440\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-k6t82b\" resourcegroup=\"rg-np-dg-minor-k6t82b-4f2hjngd28k4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-k6t82b-4f2hjngd28k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-k6t82b/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ade640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-t7jl46\" in resourcegroup=\"rg-np-version-upgrade-t7jl46-65khsnvbc4zl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t7jl46-65khsnvbc4zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t7jl46/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ade640\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-t7jl46\" in resourcegroup=\"rg-np-version-upgrade-t7jl46-65khsnvbc4zl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t7jl46-65khsnvbc4zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t7jl46/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24340\u003e: \n    failed to create NodePool np-5cw4w9: failed starting nodepool creation \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\" in resourcegroup=\"rg-nightly-4-20-b7gtl2svm55n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-b7gtl2svm55n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-5cw4w9/nodePools/np-5cw4w9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d24340\u003e: \n    failed to create NodePool np-5cw4w9: failed starting nodepool creation \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\" in resourcegroup=\"rg-nightly-4-20-b7gtl2svm55n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-b7gtl2svm55n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-5cw4w9/nodePools/np-5cw4w9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"56f51dbe4172c7698c7e2ac9b44cc9309db978fcf5349649586b91b53538f803","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"5e02d079f2bb2b4baf359fdcea15f52be4ed619abb6034bc78cd714b27a3f342","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"8bfd58413915673e86f8668f73c4a1593d6b25ac14d3df8cca7f16fa47901f61","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"c36a633bcd07dffb48200bace887b57ae6fe20eba4fb59a0da31d14814ce0b59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":15.384615384615385,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-22..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b43f40\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b43f40\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001139d00\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001139d00\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","occurred_at":"2026-09-28T23:22:35Z","signature_id":"057d78f20fc782ce9997cb17d11d93f7004a1d7a6ecbd74b197c32efd5c0100d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","occurred_at":"2026-09-28T20:21:29Z","signature_id":"b06391e0cd724ab0d438d9c0420d007165f3fe823ffd52497e5d58364ca57ca5","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-22..2026-09-28 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-q9zwpnrqm4zb/cluster-nightly-5-1-sq64f4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110cda0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-sq64f4: failed waiting for cluster=\"cluster-nightly-5-1-sq64f4\" in resourcegroup=\"rg-nightly-5-1-q9zwpnrqm4zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\",\n      \"name\": \"25287b3f-b870-4e18-a51f-514432892aec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:31:45.913312Z\",\n      \"endTime\": \"2026-09-29T00:00:52.759469958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-q9zwpnrqm4zb/cluster-nightly-5-1-sq64f4 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110cda0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-sq64f4: failed waiting for cluster=\"cluster-nightly-5-1-sq64f4\" in resourcegroup=\"rg-nightly-5-1-q9zwpnrqm4zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\",\n      \"name\": \"25287b3f-b870-4e18-a51f-514432892aec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-28T23:31:45.913312Z\",\n      \"endTime\": \"2026-09-29T00:00:52.759469958Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"d33cdf557cf6b0a812334a9c133b09038e1b46fff4e53cc99d59b39fb3a25778","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":15,"trend_counts":[1,1,1,1,2,2,1],"trend_range":"2026-09-22..2026-09-28 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":13,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f35488\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000f35488\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","signature_id":"e7f9b8ac38554f62beb05357bcb84d1c79a8ef404e325310c520ae6e5285e61d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":11,"trend_counts":[0,0,4,2,2,2,1],"trend_range":"2026-09-22..2026-09-28 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":11,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4z25qr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7lft27tlvnbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0009fab20\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4z25qr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7lft27tlvnbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc0009fab20\u003e: \n    storage account log verification failed\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104390744759341056","occurred_at":"2026-09-28T02:00:25Z","signature_id":"08f63884b8084a2606a1a5268b6fb65c1cc3e789a692cbd59f3ad65ea31fbc51","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}