{"meta":{"start_date":"2026-10-01","end_date":"2026-10-01","start_at":"2026-10-01T00:00:00Z","end_at":"2026-10-02T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T09:16:47Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":12,"failed_runs":6,"raw_occurrences":33,"matched_occurrences":33,"runs_affected":6},"rows":[{"environment":"prod","failure_pattern_id":"5de8ba5f528a28280d389eef3564abae9054b9a2afb4d837caec5c56528c05af","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":10,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":10,"after_last_push_count":10,"trend_counts":[0,0,0,0,0,0,10],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-4mwqz2\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00144a120\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-jlsd79zkk6hh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-4mwqz2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-23-4mwqz2\" to follow minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00144a120\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-jlsd79zkk6hh/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-4mwqz2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-4fwf7q\" to follow minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001226630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-wnp2jmpdpwln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-4fwf7q\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:150]: failed to remove the exact-version pin from cluster \"cluster-zstream-4-21-4fwf7q\" to follow minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001226630\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-wnp2jmpdpwln/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-4fwf7q\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-24sw5t\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000cdd5c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-vhz5sh8l55g9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-24sw5t\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-24sw5t\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000cdd5c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-vhz5sh8l55g9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-24sw5t\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"19e49d7275216187415596a77e7c4bcf8f328fed8c07e3decc611480955564e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"412c1d488719dd5fc967130e3fa6effa7e82d9edd5a17a5e8176ab4d70765e0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"4d9b0ad4d995439a69761daa6502428708fa7e25caed34c232065582abd6a8be","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"5ad91d5ba592c718110d060ec85bbce1cf9bc9da9a852c065bb122315fa608f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"7d8e65006d2598e018d984d2201d99a02258f21049c17c9d62e58cf6c40e840a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"a3f652fc0d3cf37233e1a99bcbf33f40c66eb18489b226915716ce4457c123d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"ca31150f8c635319cf607500a541ad8b66476dc3c1106a8a6a766f27645ff9cc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"d071cb6535a5544c2ea6eef944d6e2c7978595749be12c29f68a5b0ef11e7d20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"dcc2d2f958cc4b5a06713c7dd534ea32a664855ca4cc18435215b00e2e387fe7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2105477920750309376","occurred_at":"2026-10-01T02:00:28Z","signature_id":"df47e1d06972d594dea943756cc7419be51fa395cd53a06b8f50e809684c14b8","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":6,"runs_affected":4,"failed_runs":4,"run_impact_percent":33.333333333333336,"anchor_occurrences":6,"after_last_push_count":22,"trend_counts":[0,0,0,4,10,2,6],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001797968\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7fp2gpvrmpv5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbace7f5-c615-4f66-9959-02331d194dcc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbace7f5-c615-4f66-9959-02331d194dcc\",\n      \"name\": \"fbace7f5-c615-4f66-9959-02331d194dcc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:17:05.438751852Z\",\n      \"endTime\": \"2026-10-01T23:36:05.105086237Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-74m7b-t7cjp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-7fp2gpvrmpv5--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-74m7b-t7cjp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-74m7b-t7cjp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-7fp2gpvrmpv5--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-74m7b-t7cjp to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001797968\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-7fp2gpvrmpv5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbace7f5-c615-4f66-9959-02331d194dcc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbace7f5-c615-4f66-9959-02331d194dcc\",\n      \"name\": \"fbace7f5-c615-4f66-9959-02331d194dcc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:17:05.438751852Z\",\n      \"endTime\": \"2026-10-01T23:36:05.105086237Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-74m7b-t7cjp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-7fp2gpvrmpv5--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-74m7b-t7cjp to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-74m7b-t7cjp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-7fp2gpvrmpv5--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-74m7b-t7cjp to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cbe8e0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-f85th5\" in resourcegroup=\"rg-np-noedge-f85th5-2lj4rn8xdzrh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3cf330e8-6cc6-4d77-bd07-24fe8abc4160\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3cf330e8-6cc6-4d77-bd07-24fe8abc4160\",\n      \"name\": \"3cf330e8-6cc6-4d77-bd07-24fe8abc4160\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T18:02:34.984361176Z\",\n      \"endTime\": \"2026-10-01T18:21:38.6870989Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-f85th5-2lj4rn8xdzrh-np-ne-f85th5--managed/providers/Microsoft.Compute/virtualMachines/g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-f85th5-2lj4rn8xdzrh-np-ne-f85th5--managed/providers/Microsoft.Compute/virtualMachines/g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cbe8e0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-f85th5\" in resourcegroup=\"rg-np-noedge-f85th5-2lj4rn8xdzrh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3cf330e8-6cc6-4d77-bd07-24fe8abc4160\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/3cf330e8-6cc6-4d77-bd07-24fe8abc4160\",\n      \"name\": \"3cf330e8-6cc6-4d77-bd07-24fe8abc4160\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T18:02:34.984361176Z\",\n      \"endTime\": \"2026-10-01T18:21:38.6870989Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-f85th5-2lj4rn8xdzrh-np-ne-f85th5--managed/providers/Microsoft.Compute/virtualMachines/g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-f85th5-2lj4rn8xdzrh-np-ne-f85th5--managed/providers/Microsoft.Compute/virtualMachines/g4z7c0q0c5o8b3k-npnoedge-4-20-fvppx-vw46h to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010769c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-g5k9xrk5kbzf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1864d250-37c2-44c1-bbbf-426f923fe680\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1864d250-37c2-44c1-bbbf-426f923fe680\",\n      \"name\": \"1864d250-37c2-44c1-bbbf-426f923fe680\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T16:25:59.009326346Z\",\n      \"endTime\": \"2026-10-01T16:45:03.114189778Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-g5k9xrk5kbzf--managed/providers/Microsoft.Compute/virtualMachines/m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-g5k9xrk5kbzf--managed/providers/Microsoft.Compute/virtualMachines/m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010769c0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-g5k9xrk5kbzf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1864d250-37c2-44c1-bbbf-426f923fe680\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/1864d250-37c2-44c1-bbbf-426f923fe680\",\n      \"name\": \"1864d250-37c2-44c1-bbbf-426f923fe680\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T16:25:59.009326346Z\",\n      \"endTime\": \"2026-10-01T16:45:03.114189778Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-g5k9xrk5kbzf--managed/providers/Microsoft.Compute/virtualMachines/m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-g5k9xrk5kbzf--managed/providers/Microsoft.Compute/virtualMachines/m2q7s0u7z1v5z9m-arm64-vm-np-1-z7spz-94nqz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"5a88e8e35a16faf5ee7a4d5b4f6f32a8c37dfc00d34795932e2e514c6e30e085","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105687094121730048","occurred_at":"2026-10-01T15:51:39Z","signature_id":"27caddf4f1e8f34eee4fc2b6ae6d34e392067680761ae86d5ecb6266597fed04","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105687094121730048","occurred_at":"2026-10-01T15:51:39Z","signature_id":"61d0c9edd234b7e14042d3032095db877dbe6323fb17ce8fad8a175cc612fec3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105520036851486720","occurred_at":"2026-10-01T04:47:40Z","signature_id":"814a16bef1beba6789dffa5698353084a9348129da2a78e7729f185a8f59dec9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105518122772467712","occurred_at":"2026-10-01T04:40:13Z","signature_id":"ad9369c416904d0bc5cff3fb32e4116dd601fade18d109c2a14774ed4b1186d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105518122772467712","occurred_at":"2026-10-01T04:40:13Z","signature_id":"ed74568737500cd622a695c5ad75e81af359673885d49c87f038f5674080cfca","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":5,"after_last_push_count":6,"trend_counts":[0,0,0,1,0,0,5],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-47sckvjgzdjp/cluster-nightly-4-22-wwdkq6 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d874a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-wwdkq6: failed waiting for cluster=\"cluster-nightly-4-22-wwdkq6\" in resourcegroup=\"rg-nightly-4-22-47sckvjgzdjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b93cf23-0f8f-4a86-986d-b583c88b0cd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b93cf23-0f8f-4a86-986d-b583c88b0cd9\",\n      \"name\": \"4b93cf23-0f8f-4a86-986d-b583c88b0cd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:33:29.768708196Z\",\n      \"endTime\": \"2026-10-02T00:02:35.156751805Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-4-22-47sckvjgzdjp/cluster-nightly-4-22-wwdkq6 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d874a0\u003e: \n    failed to create HCP cluster cluster-nightly-4-22-wwdkq6: failed waiting for cluster=\"cluster-nightly-4-22-wwdkq6\" in resourcegroup=\"rg-nightly-4-22-47sckvjgzdjp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b93cf23-0f8f-4a86-986d-b583c88b0cd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4b93cf23-0f8f-4a86-986d-b583c88b0cd9\",\n      \"name\": \"4b93cf23-0f8f-4a86-986d-b583c88b0cd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:33:29.768708196Z\",\n      \"endTime\": \"2026-10-02T00:02:35.156751805Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-szptjh\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f34e40\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-szptjh: failed waiting for cluster=\"cp-ystream-upgrade-4-23-szptjh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-46pm69d84hvp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/780e4e8f-ebe0-42ed-942e-9333a940dc3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/780e4e8f-ebe0-42ed-942e-9333a940dc3c\",\n      \"name\": \"780e4e8f-ebe0-42ed-942e-9333a940dc3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:37:58.369447145Z\",\n      \"endTime\": \"2026-10-02T00:06:59.154886423Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-szptjh\" at install version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f34e40\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-szptjh: failed waiting for cluster=\"cp-ystream-upgrade-4-23-szptjh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-46pm69d84hvp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/780e4e8f-ebe0-42ed-942e-9333a940dc3c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/780e4e8f-ebe0-42ed-942e-9333a940dc3c\",\n      \"name\": \"780e4e8f-ebe0-42ed-942e-9333a940dc3c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T23:37:58.369447145Z\",\n      \"endTime\": \"2026-10-02T00:06:59.154886423Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-bjmvhq at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010869e0\u003e: \n    failed to create HCP cluster np-skip-minor-bjmvhq: failed waiting for cluster=\"np-skip-minor-bjmvhq\" in resourcegroup=\"rg-np-skip-minor-bjmvhq-v7kckbmqz7jj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/94714044-9af3-4f81-b892-f8e47c0efbb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/94714044-9af3-4f81-b892-f8e47c0efbb3\",\n      \"name\": \"94714044-9af3-4f81-b892-f8e47c0efbb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:05:39.688922023Z\",\n      \"endTime\": \"2026-10-02T00:34:39.296960872Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-bjmvhq at version 4.22.0-0.nightly-multi-2026-07-08-192436\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010869e0\u003e: \n    failed to create HCP cluster np-skip-minor-bjmvhq: failed waiting for cluster=\"np-skip-minor-bjmvhq\" in resourcegroup=\"rg-np-skip-minor-bjmvhq-v7kckbmqz7jj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/94714044-9af3-4f81-b892-f8e47c0efbb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/94714044-9af3-4f81-b892-f8e47c0efbb3\",\n      \"name\": \"94714044-9af3-4f81-b892-f8e47c0efbb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-02T00:05:39.688922023Z\",\n      \"endTime\": \"2026-10-02T00:34:39.296960872Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"284badc75c52f2c928896b544484ccaf81523055e480cbb01f205d32fbfb2055","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"6906776b10e91fb1930393db0f67c0e5c85ddc0131f15dc7f553da0111c50b64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"6db1e852eea8dba2acd65aead4a02f26184108da6a19027e7eac3481346d7608","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"9693157aa0c8e52231f85346439be27795bb96ffd3506957078219d7289eee55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"a47958a0b27a7e6417393acfe801cf47768b5d3548c940923347b671dfb151db","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":5,"after_last_push_count":68,"trend_counts":[5,5,0,5,5,0,5],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":14,"prior_last_seen_at":"2026-09-26T23:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f4b7a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-5rjr485pnvns/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f4b7a0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-5rjr485pnvns/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013b7a10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-55xcrpmcpjd5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013b7a10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-55xcrpmcpjd5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a6f140\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hwrdblk56ldw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000a6f140\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-hwrdblk56ldw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"0879af2742cd30b1af5adb170d9ba043c2ac72dca9e6206fd1ad2ce6a2f70845","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"167b28b3e3cf18e7829a5045c71ef1603c48da5ee1faa9e604ee810dca39cf48","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"1b8a88d9699db38c5538a15c857505c8cfc1b3d077c382a6629566c882287c82","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"9743f5915e7ad3e053054c21c8eb514f30668f1dd7765974f91761e8bad2e327","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"9bd45a89f4ba89d2fe993c417db2bf0c1d7eb96c4b27a4fc8ab08b4827e95748","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":16.666666666666668,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,0,0,2,1,0,2],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f80160\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001f80160\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012cb640\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012cb640\u003e: \n    selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105687094121730048","occurred_at":"2026-10-01T15:51:39Z","signature_id":"721a2923a68f98f8036e86221e342512b007d654edab41bc4b01d104c7c31069","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105518122772467712","occurred_at":"2026-10-01T04:40:13Z","signature_id":"42071066d578584734f0ff88f8d91844d63d17cae6fd1ba95e338cf5b297b9ff","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":2,"after_last_push_count":56,"trend_counts":[4,2,4,4,4,4,2],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":15,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed4500\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ngvxnx\" in resourcegroup=\"rg-np-version-upgrade-ngvxnx-ztnvkxg4zlcq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ngvxnx-ztnvkxg4zlcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ngvxnx/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-01-125400\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed4500\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ngvxnx\" in resourcegroup=\"rg-np-version-upgrade-ngvxnx-ztnvkxg4zlcq\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ngvxnx-ztnvkxg4zlcq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ngvxnx/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ks75xq\" for cluster \"cluster-nightly-4-20-ks75xq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012f2520\u003e: \n    failed to create NodePool np-ks75xq: failed starting nodepool creation \"np-ks75xq\" for cluster \"cluster-nightly-4-20-ks75xq\" in resourcegroup=\"rg-nightly-4-20-2zfvw4gzpm77\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-2zfvw4gzpm77/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-ks75xq/nodePools/np-ks75xq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-ks75xq\" for cluster \"cluster-nightly-4-20-ks75xq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012f2520\u003e: \n    failed to create NodePool np-ks75xq: failed starting nodepool creation \"np-ks75xq\" for cluster \"cluster-nightly-4-20-ks75xq\" in resourcegroup=\"rg-nightly-4-20-2zfvw4gzpm77\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-2zfvw4gzpm77/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-ks75xq/nodePools/np-ks75xq\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-01-125400\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"433fa8bcb3330f342d959be31de105f077c4ee1f9cf1595e004254245cb6d202","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"fb02c473ebd0d6e66f0313e3e4cc1a3da903edced25d6fe7248861c17e69533c","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":5,"trend_counts":[0,0,0,4,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000638600\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2nzmgkxwzltg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/eaa1d5d5-fc6a-4816-a207-694d27453535\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/eaa1d5d5-fc6a-4816-a207-694d27453535\",\n      \"name\": \"eaa1d5d5-fc6a-4816-a207-694d27453535\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T05:17:35.937388036Z\",\n      \"endTime\": \"2026-10-01T05:36:39.648087048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2s0w6f2w1a9o6l-np-1-wxf5k-nvwd8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2s0w6f2w1a9o6l-np-1-wxf5k-nvwd8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000638600\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-2nzmgkxwzltg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/eaa1d5d5-fc6a-4816-a207-694d27453535\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/eaa1d5d5-fc6a-4816-a207-694d27453535\",\n      \"name\": \"eaa1d5d5-fc6a-4816-a207-694d27453535\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T05:17:35.937388036Z\",\n      \"endTime\": \"2026-10-01T05:36:39.648087048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v2s0w6f2w1a9o6l-np-1-wxf5k-nvwd8: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v2s0w6f2w1a9o6l-np-1-wxf5k-nvwd8: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105520443560562688","occurred_at":"2026-10-01T04:49:27Z","signature_id":"42138565829011884507a259879669579b12c563a2f4a280f0c5c88587c8f9f4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f6d43e4ab2fe017bbc54d307366c534735e562004e9108078bd8d55ba38e56fb","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001802258\u003e: \n    resource group \"rg-zstream-upgrade-4-20-26f5nw6rjl84\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-m4xqll\" in resourcegroup=\"rg-zstream-upgrade-4-20-26f5nw6rjl84\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fdb2e512-dda1-4736-993b-cb6fd011fea5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fdb2e512-dda1-4736-993b-cb6fd011fea5\",\n      \"name\": \"fdb2e512-dda1-4736-993b-cb6fd011fea5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T17:28:43.347525555Z\",\n      \"endTime\": \"2026-10-01T17:52:46.138429989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t648pvv6ett9l7cgmh574bcs7i8oe4q still exists (deletion dispatched at 2026-10-01T17:28:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001802258\u003e: \n    resource group \"rg-zstream-upgrade-4-20-26f5nw6rjl84\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-m4xqll\" in resourcegroup=\"rg-zstream-upgrade-4-20-26f5nw6rjl84\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fdb2e512-dda1-4736-993b-cb6fd011fea5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/fdb2e512-dda1-4736-993b-cb6fd011fea5\",\n      \"name\": \"fdb2e512-dda1-4736-993b-cb6fd011fea5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T17:28:43.347525555Z\",\n      \"endTime\": \"2026-10-01T17:52:46.138429989Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t648pvv6ett9l7cgmh574bcs7i8oe4q still exists (deletion dispatched at 2026-10-01T17:28:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2105687094121730048","occurred_at":"2026-10-01T15:51:39Z","signature_id":"9cd553a21a7a83d96159492dd3d1993283938905adb80a47e77311686d9f1acf","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":8.333333333333334,"anchor_occurrences":1,"after_last_push_count":21,"trend_counts":[1,2,2,1,3,2,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":2,"prior_window_starts":["2026-09-14","2026-09-21"],"prior_runs_affected":19,"prior_last_seen_at":"2026-09-27T23:00:25Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0014996b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-10-01-083859, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0014996b0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=4.21.0-0.nightly-multi-2026-10-01-083859, targetMinor=4.22.0-0.nightly-multi-2026-07-08-192436) failed: clusterversion status.history has no version in target minor \"4.22.0-0.nightly-multi-2026-07-08-192436\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105795013534289920","occurred_at":"2026-10-01T23:00:29Z","signature_id":"d6fefeb9b4d855c8f485e15a2e9d74970a7f65266f90d3948da1d4fc0e6c9484","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}