{"meta":{"start_date":"2026-09-30","end_date":"2026-09-30","start_at":"2026-09-30T00:00:00Z","end_at":"2026-10-01T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T10:19:16Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":3,"failed_runs":2,"raw_occurrences":10,"matched_occurrences":10,"runs_affected":2},"rows":[{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":4,"after_last_push_count":54,"trend_counts":[4,4,2,4,4,4,4],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":14,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d541a0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-mv9rpr\" in resourcegroup=\"rg-np-skip-minor-mv9rpr-lccf5r2wcvcf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-mv9rpr-lccf5r2wcvcf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-mv9rpr/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d541a0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-mv9rpr\" in resourcegroup=\"rg-np-skip-minor-mv9rpr-lccf5r2wcvcf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-mv9rpr-lccf5r2wcvcf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-mv9rpr/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fzw8hf\" for cluster \"cluster-nightly-4-20-fzw8hf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca4c80\u003e: \n    failed to create NodePool np-fzw8hf: failed starting nodepool creation \"np-fzw8hf\" for cluster \"cluster-nightly-4-20-fzw8hf\" in resourcegroup=\"rg-nightly-4-20-tfqsvtwn4ns7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-tfqsvtwn4ns7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-fzw8hf/nodePools/np-fzw8hf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fzw8hf\" for cluster \"cluster-nightly-4-20-fzw8hf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ca4c80\u003e: \n    failed to create NodePool np-fzw8hf: failed starting nodepool creation \"np-fzw8hf\" for cluster \"cluster-nightly-4-20-fzw8hf\" in resourcegroup=\"rg-nightly-4-20-tfqsvtwn4ns7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-tfqsvtwn4ns7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-fzw8hf/nodePools/np-fzw8hf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-014938\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001472300\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4zkcv5\" resourcegroup=\"rg-np-dg-minor-4zkcv5-pjsq2w8jb2dl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4zkcv5-pjsq2w8jb2dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4zkcv5/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-30-185324\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001472300\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4zkcv5\" resourcegroup=\"rg-np-dg-minor-4zkcv5-pjsq2w8jb2dl\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4zkcv5-pjsq2w8jb2dl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4zkcv5/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-30-185324\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","signature_id":"5d856dedaa58bb923d9cef2471b19783816a8d29d3cf957e7044257ee539a42c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","signature_id":"6313ae0584d2d0c419cdb14a4f57c8d3049e608c9cfe03eac9cc76d52450b807","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","signature_id":"9b73350e92406053dbb562adaf95183f694e1699162837a7daf5336133bff3ca","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","signature_id":"deca178dbaa2cc480c8418c70ba18703fd89f0904bca91f4e92e8446fbaf30cc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":2,"after_last_push_count":16,"trend_counts":[0,0,0,0,4,10,2],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013120e0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sttxg5n27hh5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95422086-7eb5-4fab-930f-f78b5c9185bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95422086-7eb5-4fab-930f-f78b5c9185bc\",\n      \"name\": \"95422086-7eb5-4fab-930f-f78b5c9185bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T00:05:50.759671414Z\",\n      \"endTime\": \"2026-10-01T00:24:59.018405341Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sttxg5n27hh5--managed/providers/Microsoft.Compute/virtualMachines/m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sttxg5n27hh5--managed/providers/Microsoft.Compute/virtualMachines/m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013120e0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-sttxg5n27hh5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95422086-7eb5-4fab-930f-f78b5c9185bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/95422086-7eb5-4fab-930f-f78b5c9185bc\",\n      \"name\": \"95422086-7eb5-4fab-930f-f78b5c9185bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T00:05:50.759671414Z\",\n      \"endTime\": \"2026-10-01T00:24:59.018405341Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sttxg5n27hh5--managed/providers/Microsoft.Compute/virtualMachines/m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-sttxg5n27hh5--managed/providers/Microsoft.Compute/virtualMachines/m8y8w5f8k4s4f3b-autoscale-noaz-xcklm-vq2zl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c85c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-wbpxbvspwkdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/44b3f004-e16f-436f-bdc7-4652956b17b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/44b3f004-e16f-436f-bdc7-4652956b17b0\",\n      \"name\": \"44b3f004-e16f-436f-bdc7-4652956b17b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:12:50.954793928Z\",\n      \"endTime\": \"2026-09-30T02:31:50.125456331Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-mhjdk-jdk87: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wbpxbvspwkdl--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mhjdk-jdk87 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-mhjdk-jdk87: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wbpxbvspwkdl--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mhjdk-jdk87 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009c85c0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-wbpxbvspwkdl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/44b3f004-e16f-436f-bdc7-4652956b17b0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/44b3f004-e16f-436f-bdc7-4652956b17b0\",\n      \"name\": \"44b3f004-e16f-436f-bdc7-4652956b17b0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-30T02:12:50.954793928Z\",\n      \"endTime\": \"2026-09-30T02:31:50.125456331Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-mhjdk-jdk87: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wbpxbvspwkdl--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mhjdk-jdk87 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-mhjdk-jdk87: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-wbpxbvspwkdl--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-mhjdk-jdk87 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","signature_id":"6342306aa2c9342124f24885b42800960540bcf682af76df09c7083cf26dad88","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105115529034338304","occurred_at":"2026-09-30T02:00:27Z","signature_id":"2f89faaaf7b43a2918fd4eba43a3db8eedeb08cfc96d4a5da653b1792d8738d4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":66.66666666666667,"anchor_occurrences":2,"after_last_push_count":20,"trend_counts":[1,1,2,2,1,3,2],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":18,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:178]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001069ba8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-29-171841, targetMinor=5.1.0-0.nightly-multi-2026-09-30-105558) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-30-105558\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:178]: Timed out after 2700.007s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001069ba8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-29-171841, targetMinor=5.1.0-0.nightly-multi-2026-09-30-105558) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-30-105558\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001415050\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc001415050\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","signature_id":"a4aa188ac160ec2ff4c13c20c75a17504feb16474f727e3f622fd64eae9523cf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105115529034338304","occurred_at":"2026-09-30T02:00:27Z","signature_id":"342894fb948ca9c214cc8f0f4cb97e686763db2288c64ee0b0f2635fc366a42d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":33.333333333333336,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,1,2],"trend_range":"2026-09-24..2026-09-30 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to verify HCP cluster viability for patch-tags test\nUnexpected error:\n    \u003c*errors.joinError | 0xc001619020\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:198]: failed to verify HCP cluster viability for patch-tags test\nUnexpected error:\n    \u003c*errors.joinError | 0xc001619020\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-5-0-b96xbr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e76f48\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-5-0-b96xbr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc000e76f48\u003e: \n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","signature_id":"2abb3bce100e636041d87fa3d6c3423531ab2ba3b2d69922e70f8a53d50142a2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105432621860261888","occurred_at":"2026-09-30T23:00:28Z","signature_id":"733cfb2369652cdeae179de94ce49731d6a1d3599568a270c3308ae212724dfa","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}