{"meta":{"start_date":"2026-10-01","end_date":"2026-10-01","start_at":"2026-10-01T00:00:00Z","end_at":"2026-10-02T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T09:40:03Z","environments":["int"]},"environments":[{"environment":"int","summary":{"total_runs":6,"failed_runs":4,"raw_occurrences":34,"matched_occurrences":34,"runs_affected":4},"rows":[{"environment":"int","failure_pattern_id":"d1ee456764b76847fb37d4ebdfa7f922f314e4cd07162e412d7b74bfc70ff08f","failure_pattern":"ERROR CODE: InvalidNullTagValue; detail message Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidNullTagValue","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":30,"runs_affected":3,"failed_runs":3,"run_impact_percent":50,"anchor_occurrences":30,"after_last_push_count":40,"trend_counts":[0,0,0,0,0,10,30],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-fpkhgg\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013695c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-x2qm64ntnf4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-fpkhgg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:169]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-fpkhgg\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013695c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-x2qm64ntnf4k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-fpkhgg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-5lppxg\" on minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c77c20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-9qkw2hc8j64k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5lppxg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-5lppxg\" on minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000c77c20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-9qkw2hc8j64k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-5lppxg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-5jgcvl\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00156e510\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-qqh9mh2dgv5d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-5jgcvl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:156]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-5jgcvl\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc00156e510\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-qqh9mh2dgv5d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-5jgcvl\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidNullTagValue\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidNullTagValue\",\n        \"message\": \"Invalid tag value. The following tags 'aro-hcp.experimental.cluster.control-plane-exact-version' have a null value. Tag value cannot be null.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"09ad1536393e5c2e93d734e6e4a7baabe569c00b7a346b81e194a1dc526d6c84","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"1e70068ca83e3da2caa181c14b566633dee772461c6bbef7e3b3c3b11c58ede2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"53d5243f6499b6bbb1f41dd260ea791225c721f0e2219068f0049f16fa9044cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"881e0d266cdbc141cde714bffad14ec487d883af4eafed8f23bce4c4912d543e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"8928cf54b630d0e6361d33389106d1e4b00e5b4fdda04424c7b1b3c9b344e25a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"93276e8c2e659949c04804f296318e19a7261db922d78b2b7a491384b13155c9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"af5e31a50e667e6789ed1a04349bf851557b9188ce2f9b1e9df3b6b675fccd7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"e47528e4415d38ba10698d98e1ae80c6cfa1a4632c798890c9c25df89d4ae02f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"ee5d11d2818efb7c3c4ed23a4e01d5e6ab980563ce83dc358792a3f97e2d1f1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105606654199336960","occurred_at":"2026-10-01T10:32:01Z","signature_id":"fae23cdbef205ba19e74bef0e349a708114fea21f0bfce452cbd47ff26764f2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"3a4c53c8b8ed0f4e3a8c8625b638982616ccea702a79f9698e9303da79a15f67","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"3f239f77699bb84da05b2be013b74a288bb4538f8ac6f15ec4ea749a60735ad4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"4b7a489ef3ea65128a354e82da4f4e1de6bc22ca63444681abe780ccc7ddf4d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"9ed87e9e9315982414d4d556eba0d0bea3967a50c31ea93eff6a36c532a8eb2a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"a23758f9b4de17fd810383f0e57b42b4438075c36125aa9b6c712edcb5ea75a5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"a5ad9b8fa8feb1257e28216d87d1a08d738224a79e57266bf4690f58f661ad42","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"ad3a6f95d691523f560574aa964dea297931db1f1651191d1fcc28b85649a55a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"b42e83d1c1e188b5a53a98054e2e876e0d0416f2e4658e9a37a24c36e12854fe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"f393133e89a119909f1414ed37aedeb249de63da9461ad1d5feb1022be0d1b09","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105558232767401984","occurred_at":"2026-10-01T07:19:36Z","signature_id":"ffa6512d457f2bdedb8df8b3e86503ab5c120285bf88c97a42b1b851b87335e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"0ec303987eddb2fdc7951010bee64e64f19b2795c36b27fa9b6c16d511039f6d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"10b16b7588fc77933a91ee23f48c974f59b3cf6c7796512f1986ba476c2fdcba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"535d9b5810530eb8fa93d005fe5342bc6c5f76b1ac4619d0de9e457c08ad121a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"652af8ac6d7caa46ada925ea05e4b4f678aa2fa90a5c6290737d7b660b7f2248","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"a2c9210be7e28a9733e941546fc75b632c067fbedccb9d0a6c5fc389f9a2e48f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"a8535460c6d3cd45df9df1883152d24842155b835833878a66f7b2d3ba6aaa58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"bc6beae7f23eaeb00f646e9afe501ded4abd7941cafe250a38a63603361caf1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"bc99803c563848f8ef2751f1a43b94eaeabe40a9b89129e706b11fea21a2a78d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"de5390b7de0f9fb6b5b999f0840b119c4cfcea4837064e22c1a37c81aa8301d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"eda6d7fa1b7ee822da5e9598ea4cc8fe8832dbc41c8d88d803855760b7f64988","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3,"runs_affected":2,"failed_runs":2,"run_impact_percent":33.333333333333336,"anchor_occurrences":3,"after_last_push_count":11,"trend_counts":[0,0,1,1,4,2,3],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-27T07:04:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c9e80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fxmrv96q875w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\",\n      \"name\": \"a1048b90-244a-4c5a-ba89-e1135a68a269\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T20:23:38.274618399Z\",\n      \"endTime\": \"2026-10-01T20:42:38.037012934Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0012c9e80\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-fxmrv96q875w\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1048b90-244a-4c5a-ba89-e1135a68a269\",\n      \"name\": \"a1048b90-244a-4c5a-ba89-e1135a68a269\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T20:23:38.274618399Z\",\n      \"endTime\": \"2026-10-01T20:42:38.037012934Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-26bmw to exist\\nMachine y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-fxmrv96q875w--managed/providers/Microsoft.Compute/virtualMachines/y7y7b3o0h7v8m9j-np-1-tdqpw-dq6sk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000941160\u003e: \n    failed to create NodePool np-jv9hvk: failed waiting for nodepool=\"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\" in resourcegroup=\"rg-candidate-4-20-6nfp6mj4gt68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n      \"name\": \"29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T20:46:24.926645724Z\",\n      \"endTime\": \"2026-10-01T21:05:28.967720321Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000941160\u003e: \n    failed to create NodePool np-jv9hvk: failed waiting for nodepool=\"np-jv9hvk\" for cluster \"cluster-candidate-4-20-jv9hvk\" in resourcegroup=\"rg-candidate-4-20-6nfp6mj4gt68\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n      \"name\": \"29d69384-8f10-43a3-adab-ad5ed046a0d7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T20:46:24.926645724Z\",\n      \"endTime\": \"2026-10-01T21:05:28.967720321Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-6nfp6mj4gt68-candidate-jv9hvk--managed/providers/Microsoft.Compute/virtualMachines/f9f2c2u0c2w2q1m-np-jv9hvk-rcvg7-q6krr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00057dcc0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jspphct49kmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\",\n      \"name\": \"4a191606-d1f9-418e-b590-0307f5435adb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T01:58:13.545499376Z\",\n      \"endTime\": \"2026-10-01T02:17:18.498756095Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:109]: failed to create ARM64 nodepool \"arm64-vm-np-1\" with VM size \"Standard_D2plds_v6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00057dcc0\u003e: \n    failed to create NodePool arm64-vm-np-1: failed waiting for nodepool=\"arm64-vm-np-1\" for cluster \"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-jspphct49kmw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a191606-d1f9-418e-b590-0307f5435adb\",\n      \"name\": \"4a191606-d1f9-418e-b590-0307f5435adb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T01:58:13.545499376Z\",\n      \"endTime\": \"2026-10-01T02:17:18.498756095Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-jspphct49kmw--managed/providers/Microsoft.Compute/virtualMachines/x9l9x8u4y2b5f7r-arm64-vm-np-1-lt5pz-7ckxj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105746340389588992","occurred_at":"2026-10-01T19:47:05Z","signature_id":"24417d5ef077cde226947335837e6d950a92fb685e28c57995b46f4508d8e565","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105746340389588992","occurred_at":"2026-10-01T19:47:05Z","signature_id":"a7f9c67af898546f71a808604e7639e1ebc1c0ed5391f8286fb71f6bda1d5873","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"e33b49f25d1ec8838fb3fb2248feed144429268d006904054aadad3f41b2c4ef","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"b3f3e0a11bd87e1d792b79fd0f26a19cb61cf8deaa26da50a5b798fb45c47433","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,1,0,0,0,0,1],"trend_range":"2026-09-25..2026-10-01 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-21"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-26T21:16:00Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109d0c0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s7p6hptqhbvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\",\n      \"name\": \"357a31b9-a87e-4b62-a8b5-196b066234f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T02:58:38.669148771Z\",\n      \"endTime\": \"2026-10-01T03:17:44.618360897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00109d0c0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-s7p6hptqhbvb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/357a31b9-a87e-4b62-a8b5-196b066234f1\",\n      \"name\": \"357a31b9-a87e-4b62-a8b5-196b066234f1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-01T02:58:38.669148771Z\",\n      \"endTime\": \"2026-10-01T03:17:44.618360897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine m5a5s2c3g7w1p0l-autoscale-noaz-8xp74-xzdr6: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105471177496465408","occurred_at":"2026-10-01T01:33:40Z","signature_id":"44b6707750ec56079daaa0398aa289b9339df5e0faca1b38d9f04663d954b374","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}