{"meta":{"date":"2026-10-07","timezone":"UTC","generated_at":"2026-10-30T18:15:46Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":7,"failed_runs":7,"runs_with_occurrences":7,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-07T23:33:31Z","started_at":"2026-10-07T23:33:31Z","completed_at":"2026-10-08T02:24:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T02:34:13.083923279Z","timing_metadata_checked_at":"2026-10-08T02:34:13.083923279Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T02:34:13.088006164Z","region_metadata_checked_at":"2026-10-08T02:34:13.088006164Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"12caeaae2816610c5e4cb1b4409b4bdc62384b53973b15e7253ddcab50460f06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"4e1cdbd87ab7f2945023de5abed108f7e637f867d4aa7849637d211a4e777718","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-94cskf\" for cluster \"cluster-candidate-4-20-94cskf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14001684f520\u003e: \n    failed to create NodePool np-94cskf: failed waiting for nodepool=\"np-94cskf\" for cluster \"cluster-candidate-4-20-94cskf\" in resourcegroup=\"rg-candidate-4-20-dx6vntb24jx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ab0c46ad-c2da-4d9c-a097-c7258ebd8294\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ab0c46ad-c2da-4d9c-a097-c7258ebd8294\",\n      \"name\": \"ab0c46ad-c2da-4d9c-a097-c7258ebd8294\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:37:45.188343344Z\",\n      \"endTime\": \"2026-10-08T01:06:45.910047845Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q3z7q8q2v9s9t7g-np-94cskf-hpd2z-bpg28: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-dx6vntb24jx5-candidate-94cskf--managed/providers/Microsoft.Compute/virtualMachines/q3z7q8q2v9s9t7g-np-94cskf-hpd2z-bpg28 to exist\\nMachine q3z7q8q2v9s9t7g-np-94cskf-hpd2z-c6965: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-dx6vntb24jx5-candidate-94cskf--managed/providers/Microsoft.Compute/virtualMachines/q3z7q8q2v9s9t7g-np-94cskf-hpd2z-c6965 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q3z7q8q2v9s9t7g-np-94cskf-hpd2z-bpg28: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-dx6vntb24jx5-candidate-94cskf--managed/providers/Microsoft.Compute/virtualMachines/q3z7q8q2v9s9t7g-np-94cskf-hpd2z-bpg28 to exist\\nMachine q3z7q8q2v9s9t7g-np-94cskf-hpd2z-c6965: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-dx6vntb24jx5-candidate-94cskf--managed/providers/Microsoft.Compute/virtualMachines/q3z7q8q2v9s9t7g-np-94cskf-hpd2z-c6965 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-94cskf\" for cluster \"cluster-candidate-4-20-94cskf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14001684f520\u003e: \n    failed to create NodePool np-94cskf: failed waiting for nodepool=\"np-94cskf\" for cluster \"cluster-candidate-4-20-94cskf\" in resourcegroup=\"rg-candidate-4-20-dx6vntb24jx5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ab0c46ad-c2da-4d9c-a097-c7258ebd8294\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ab0c46ad-c2da-4d9c-a097-c7258ebd8294\",\n      \"name\": \"ab0c46ad-c2da-4d9c-a097-c7258ebd8294\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:37:45.188343344Z\",\n      \"endTime\": \"2026-10-08T01:06:45.910047845Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q3z7q8q2v9s9t7g-np-94cskf-hpd2z-bpg28: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-dx6vntb24jx5-candidate-94cskf--managed/providers/Microsoft.Compute/virtualMachines/q3z7q8q2v9s9t7g-np-94cskf-hpd2z-bpg28 to exist\\nMachine q3z7q8q2v9s9t7g-np-94cskf-hpd2z-c6965: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-dx6vntb24jx5-candidate-94cskf--managed/providers/Microsoft.Compute/virtualMachines/q3z7q8q2v9s9t7g-np-94cskf-hpd2z-c6965 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q3z7q8q2v9s9t7g-np-94cskf-hpd2z-bpg28: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-dx6vntb24jx5-candidate-94cskf--managed/providers/Microsoft.Compute/virtualMachines/q3z7q8q2v9s9t7g-np-94cskf-hpd2z-bpg28 to exist\\nMachine q3z7q8q2v9s9t7g-np-94cskf-hpd2z-c6965: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-dx6vntb24jx5-candidate-94cskf--managed/providers/Microsoft.Compute/virtualMachines/q3z7q8q2v9s9t7g-np-94cskf-hpd2z-c6965 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"12d104d96a3ec1844bd8a354d037f4c7bd930cbd1d43681399fd73b45b51a0a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"d9a1da04dc652deaf64a62fa6192a041922c1cfe9c39fc4d3a8476773a01a71c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6dggkt\" for cluster \"cluster-candidate-4-21-6dggkt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b3ad6d92ec0\u003e: \n    failed to create NodePool np-6dggkt: failed waiting for nodepool=\"np-6dggkt\" for cluster \"cluster-candidate-4-21-6dggkt\" in resourcegroup=\"rg-candidate-4-21-jgv8rv8mnpsq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c878b86a-b2d7-468c-b2a2-4e3f590ed7b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c878b86a-b2d7-468c-b2a2-4e3f590ed7b4\",\n      \"name\": \"c878b86a-b2d7-468c-b2a2-4e3f590ed7b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:37:43.744141421Z\",\n      \"endTime\": \"2026-10-08T01:06:45.988967998Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-7tx5x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-jgv8rv8mnpsq-candidate-6dggkt--managed/providers/Microsoft.Compute/virtualMachines/r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-7tx5x to exist\\nMachine r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-b5dkg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-jgv8rv8mnpsq-candidate-6dggkt--managed/providers/Microsoft.Compute/virtualMachines/r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-b5dkg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-7tx5x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-jgv8rv8mnpsq-candidate-6dggkt--managed/providers/Microsoft.Compute/virtualMachines/r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-7tx5x to exist\\nMachine r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-b5dkg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-jgv8rv8mnpsq-candidate-6dggkt--managed/providers/Microsoft.Compute/virtualMachines/r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-b5dkg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6dggkt\" for cluster \"cluster-candidate-4-21-6dggkt\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b3ad6d92ec0\u003e: \n    failed to create NodePool np-6dggkt: failed waiting for nodepool=\"np-6dggkt\" for cluster \"cluster-candidate-4-21-6dggkt\" in resourcegroup=\"rg-candidate-4-21-jgv8rv8mnpsq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c878b86a-b2d7-468c-b2a2-4e3f590ed7b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c878b86a-b2d7-468c-b2a2-4e3f590ed7b4\",\n      \"name\": \"c878b86a-b2d7-468c-b2a2-4e3f590ed7b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:37:43.744141421Z\",\n      \"endTime\": \"2026-10-08T01:06:45.988967998Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-7tx5x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-jgv8rv8mnpsq-candidate-6dggkt--managed/providers/Microsoft.Compute/virtualMachines/r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-7tx5x to exist\\nMachine r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-b5dkg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-jgv8rv8mnpsq-candidate-6dggkt--managed/providers/Microsoft.Compute/virtualMachines/r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-b5dkg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-7tx5x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-jgv8rv8mnpsq-candidate-6dggkt--managed/providers/Microsoft.Compute/virtualMachines/r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-7tx5x to exist\\nMachine r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-b5dkg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-jgv8rv8mnpsq-candidate-6dggkt--managed/providers/Microsoft.Compute/virtualMachines/r3e0a4l3f2r4r2r-np-6dggkt-d8bnk-b5dkg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"16029a60c325f9c801b9f0238ac6b4a05bf962beba9d4344327e6e4809ccddf3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"89dcd307570448e824d023284861443a0dfbff7e5115cf1259f0fe9ffd4aa969","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x25da79d21660\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2wld6d6dj2tk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fc4a1e49-a3dd-434a-b2ad-6f784f4be0d8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fc4a1e49-a3dd-434a-b2ad-6f784f4be0d8\",\n      \"name\": \"fc4a1e49-a3dd-434a-b2ad-6f784f4be0d8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:08:52.396209625Z\",\n      \"endTime\": \"2026-10-08T00:38:01.112273549Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5l3c3n7u0v9d0a-np-1-lks62-vc5ds: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-2wld6d6dj2tk--managed/providers/Microsoft.Compute/virtualMachines/b5l3c3n7u0v9d0a-np-1-lks62-vc5ds to exist\\nMachine b5l3c3n7u0v9d0a-np-1-lks62-wcnmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-2wld6d6dj2tk--managed/providers/Microsoft.Compute/virtualMachines/b5l3c3n7u0v9d0a-np-1-lks62-wcnmt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5l3c3n7u0v9d0a-np-1-lks62-vc5ds: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-2wld6d6dj2tk--managed/providers/Microsoft.Compute/virtualMachines/b5l3c3n7u0v9d0a-np-1-lks62-vc5ds to exist\\nMachine b5l3c3n7u0v9d0a-np-1-lks62-wcnmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-2wld6d6dj2tk--managed/providers/Microsoft.Compute/virtualMachines/b5l3c3n7u0v9d0a-np-1-lks62-wcnmt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x25da79d21660\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-2wld6d6dj2tk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fc4a1e49-a3dd-434a-b2ad-6f784f4be0d8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/fc4a1e49-a3dd-434a-b2ad-6f784f4be0d8\",\n      \"name\": \"fc4a1e49-a3dd-434a-b2ad-6f784f4be0d8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:08:52.396209625Z\",\n      \"endTime\": \"2026-10-08T00:38:01.112273549Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5l3c3n7u0v9d0a-np-1-lks62-vc5ds: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-2wld6d6dj2tk--managed/providers/Microsoft.Compute/virtualMachines/b5l3c3n7u0v9d0a-np-1-lks62-vc5ds to exist\\nMachine b5l3c3n7u0v9d0a-np-1-lks62-wcnmt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-2wld6d6dj2tk--managed/providers/Microsoft.Compute/virtualMachines/b5l3c3n7u0v9d0a-np-1-lks62-wcnmt to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5l3c3n7u0v9d0a-np-1-lks62-vc5ds: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-2wld6d6dj2tk--managed/providers/Microsoft.Compute/virtualMachines/b5l3c3n7u0v9d0a-np-1-lks62-vc5ds to exist\\nMachine b5l3c3n7u0v9d0a-np-1-lks62-wcnmt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-2wld6d6dj2tk--managed/providers/Microsoft.Compute/virtualMachines/b5l3c3n7u0v9d0a-np-1-lks62-wcnmt to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"35767c2c1d43aa3c0b24667120a6f7b905b7c93643726335beaaf385992af348","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"2a467c8dce60d0e129d691c98b8512c8ec1ed634b0dd9e1fd39a756686e7121b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tlv9nh\" for cluster \"cluster-candidate-4-22-tlv9nh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36df90d3b540\u003e: \n    failed to create NodePool np-tlv9nh: failed waiting for nodepool=\"np-tlv9nh\" for cluster \"cluster-candidate-4-22-tlv9nh\" in resourcegroup=\"rg-candidate-4-22-k8bbp9vgvcln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b294c96-939a-4202-a8fa-48f02c92df7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b294c96-939a-4202-a8fa-48f02c92df7e\",\n      \"name\": \"5b294c96-939a-4202-a8fa-48f02c92df7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:39:43.973177646Z\",\n      \"endTime\": \"2026-10-08T01:08:45.917568185Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd to exist\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd to exist\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tlv9nh\" for cluster \"cluster-candidate-4-22-tlv9nh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36df90d3b540\u003e: \n    failed to create NodePool np-tlv9nh: failed waiting for nodepool=\"np-tlv9nh\" for cluster \"cluster-candidate-4-22-tlv9nh\" in resourcegroup=\"rg-candidate-4-22-k8bbp9vgvcln\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b294c96-939a-4202-a8fa-48f02c92df7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b294c96-939a-4202-a8fa-48f02c92df7e\",\n      \"name\": \"5b294c96-939a-4202-a8fa-48f02c92df7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:39:43.973177646Z\",\n      \"endTime\": \"2026-10-08T01:08:45.917568185Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd to exist\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-mgfpd to exist\\nMachine g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-k8bbp9vgvcln-candidate-tlv9nh--managed/providers/Microsoft.Compute/virtualMachines/g0k7s1c1z6e9t3k-np-tlv9nh-lbkj4-nq27w to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3d9c8bc3436d657af1f13910f34ba5d553c4f7d1441496a10f0ae6d540845d0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"d5adfea4d2420279717d9c7a8651e85415083eda53a1fc1cec731d467be31793","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2c512700e018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bsqjlrm7bfgr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/597e749b-3cff-43ff-bf7c-19c6478c2f1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/597e749b-3cff-43ff-bf7c-19c6478c2f1e\",\n      \"name\": \"597e749b-3cff-43ff-bf7c-19c6478c2f1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:07:05.991580079Z\",\n      \"endTime\": \"2026-10-08T00:36:11.101154752Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-6rdnb-gdvcq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-gdvcq to exist\\nMachine cilium-cluster-cilium-np-6rdnb-x7dhr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-x7dhr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-6rdnb-gdvcq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-gdvcq to exist\\nMachine cilium-cluster-cilium-np-6rdnb-x7dhr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-x7dhr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2c512700e018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-bsqjlrm7bfgr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/597e749b-3cff-43ff-bf7c-19c6478c2f1e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/597e749b-3cff-43ff-bf7c-19c6478c2f1e\",\n      \"name\": \"597e749b-3cff-43ff-bf7c-19c6478c2f1e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:07:05.991580079Z\",\n      \"endTime\": \"2026-10-08T00:36:11.101154752Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-6rdnb-gdvcq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-gdvcq to exist\\nMachine cilium-cluster-cilium-np-6rdnb-x7dhr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-x7dhr to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-6rdnb-gdvcq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-gdvcq to exist\\nMachine cilium-cluster-cilium-np-6rdnb-x7dhr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-bsqjlrm7bfgr--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-6rdnb-x7dhr to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"4e893b80840ba0ac3c23cb77b8f84e14cbb315598cb2a9a3a445cc064b217f49","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"0df775ba92680196995c3176cc7ea2cf943ff930e220b5800c510ebe552cb1a2","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3133a1380440\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-dfm6bjrfc7jr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30ebea1e-66ce-4e21-ae0f-88cedbdc4981\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30ebea1e-66ce-4e21-ae0f-88cedbdc4981\",\n      \"name\": \"30ebea1e-66ce-4e21-ae0f-88cedbdc4981\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:07:37.937984296Z\",\n      \"endTime\": \"2026-10-08T00:36:41.067847774Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-nb26h-jsv7r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dfm6bjrfc7jr--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-nb26h-jsv7r to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-nb26h-jsv7r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dfm6bjrfc7jr--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-nb26h-jsv7r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3133a1380440\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-dfm6bjrfc7jr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30ebea1e-66ce-4e21-ae0f-88cedbdc4981\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30ebea1e-66ce-4e21-ae0f-88cedbdc4981\",\n      \"name\": \"30ebea1e-66ce-4e21-ae0f-88cedbdc4981\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:07:37.937984296Z\",\n      \"endTime\": \"2026-10-08T00:36:41.067847774Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-nb26h-jsv7r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dfm6bjrfc7jr--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-nb26h-jsv7r to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-nb26h-jsv7r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-dfm6bjrfc7jr--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-nb26h-jsv7r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"607914fa1f2f363109c38acb8be1bece25afb333dce32c16d9d529e9f4026d54","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"ad8aa7d0e446ca97b82f47c75fb4959ef7171a8fc84389bd39c3be3472d4a550","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x32955c70b520\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-f92csqngd5f6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c45e69d4-f127-4c8c-84bd-5e3ef22ca821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c45e69d4-f127-4c8c-84bd-5e3ef22ca821\",\n      \"name\": \"c45e69d4-f127-4c8c-84bd-5e3ef22ca821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:32:50.695442989Z\",\n      \"endTime\": \"2026-10-08T02:01:50.773194309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x to exist\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x to exist\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x32955c70b520\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-f92csqngd5f6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c45e69d4-f127-4c8c-84bd-5e3ef22ca821\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c45e69d4-f127-4c8c-84bd-5e3ef22ca821\",\n      \"name\": \"c45e69d4-f127-4c8c-84bd-5e3ef22ca821\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T01:32:50.695442989Z\",\n      \"endTime\": \"2026-10-08T02:01:50.773194309Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x to exist\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-psf6x to exist\\nMachine o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f92csqngd5f6--managed/providers/Microsoft.Compute/virtualMachines/o4r3a3f0l7z2h4b-np-deletable-52qkj-qqxfj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6xhlmlzjgvst\" to finish creating: GET https://management"}},{"row_id":"72d26631f2376c199729278bc928c85c5ff8fc505003784f10938658e2966952","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"50f6bef9fb00580c65f782fcadba664ddc8335bf13557695ef58074a611962dd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dbzlxp\" for cluster \"cluster-candidate-4-23-dbzlxp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3262c3fbda60\u003e: \n    failed to create NodePool np-dbzlxp: failed waiting for nodepool=\"np-dbzlxp\" for cluster \"cluster-candidate-4-23-dbzlxp\" in resourcegroup=\"rg-candidate-4-23-7mxgwdtr4c9g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cae9b2c8-0b30-4246-ac5e-55bae91ac247\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cae9b2c8-0b30-4246-ac5e-55bae91ac247\",\n      \"name\": \"cae9b2c8-0b30-4246-ac5e-55bae91ac247\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:39:22.908278735Z\",\n      \"endTime\": \"2026-10-08T01:08:25.937295486Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-5lwsr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-7mxgwdtr4c9g-candidate-dbzlxp--managed/providers/Microsoft.Compute/virtualMachines/q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-5lwsr to exist\\nMachine q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-vqfkj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-7mxgwdtr4c9g-candidate-dbzlxp--managed/providers/Microsoft.Compute/virtualMachines/q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-vqfkj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-5lwsr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-7mxgwdtr4c9g-candidate-dbzlxp--managed/providers/Microsoft.Compute/virtualMachines/q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-5lwsr to exist\\nMachine q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-vqfkj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-7mxgwdtr4c9g-candidate-dbzlxp--managed/providers/Microsoft.Compute/virtualMachines/q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-vqfkj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dbzlxp\" for cluster \"cluster-candidate-4-23-dbzlxp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3262c3fbda60\u003e: \n    failed to create NodePool np-dbzlxp: failed waiting for nodepool=\"np-dbzlxp\" for cluster \"cluster-candidate-4-23-dbzlxp\" in resourcegroup=\"rg-candidate-4-23-7mxgwdtr4c9g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cae9b2c8-0b30-4246-ac5e-55bae91ac247\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/cae9b2c8-0b30-4246-ac5e-55bae91ac247\",\n      \"name\": \"cae9b2c8-0b30-4246-ac5e-55bae91ac247\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:39:22.908278735Z\",\n      \"endTime\": \"2026-10-08T01:08:25.937295486Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-5lwsr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-7mxgwdtr4c9g-candidate-dbzlxp--managed/providers/Microsoft.Compute/virtualMachines/q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-5lwsr to exist\\nMachine q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-vqfkj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-7mxgwdtr4c9g-candidate-dbzlxp--managed/providers/Microsoft.Compute/virtualMachines/q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-vqfkj to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-5lwsr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-7mxgwdtr4c9g-candidate-dbzlxp--managed/providers/Microsoft.Compute/virtualMachines/q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-5lwsr to exist\\nMachine q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-vqfkj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-7mxgwdtr4c9g-candidate-dbzlxp--managed/providers/Microsoft.Compute/virtualMachines/q2d3x1j4c1h2j3i-np-dbzlxp-ngqkn-vqfkj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7ab8649ea32938b7f6cb3470f74541db9c02a0a9f40d2cf7f6dcc694de0f623e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"016fceb8857335fe4989864c4eb9b68117f9071b521a6c986d09b960f5ee4cc7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gcl76m\" for cluster \"cluster-candidate-5-0-gcl76m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ac110093d00\u003e: \n    failed to create NodePool np-gcl76m: failed waiting for nodepool=\"np-gcl76m\" for cluster \"cluster-candidate-5-0-gcl76m\" in resourcegroup=\"rg-candidate-5-0-fvphxw8q5b7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/891d054c-0f91-44b3-8882-6aa82d1f6e9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/891d054c-0f91-44b3-8882-6aa82d1f6e9b\",\n      \"name\": \"891d054c-0f91-44b3-8882-6aa82d1f6e9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:42:35.095507926Z\",\n      \"endTime\": \"2026-10-08T01:11:35.931061942Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z to exist\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z to exist\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-gcl76m\" for cluster \"cluster-candidate-5-0-gcl76m\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ac110093d00\u003e: \n    failed to create NodePool np-gcl76m: failed waiting for nodepool=\"np-gcl76m\" for cluster \"cluster-candidate-5-0-gcl76m\" in resourcegroup=\"rg-candidate-5-0-fvphxw8q5b7n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/891d054c-0f91-44b3-8882-6aa82d1f6e9b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/891d054c-0f91-44b3-8882-6aa82d1f6e9b\",\n      \"name\": \"891d054c-0f91-44b3-8882-6aa82d1f6e9b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:42:35.095507926Z\",\n      \"endTime\": \"2026-10-08T01:11:35.931061942Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z to exist\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-gx48z to exist\\nMachine i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-fvphxw8q5b7n-candidate-gcl76m--managed/providers/Microsoft.Compute/virtualMachines/i0y0d7o1e9p6l7u-np-gcl76m-jkvcw-srzbd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"898011a1202b3076cbe9d269994225d52cb3ef08aede9a3401ae38cd613ef3ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"c77f01f179024436a11b7e008e5544b959c8dce85fa7b4baaf4c0b19ae4e63ad","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tjjkwl\" for cluster \"cluster-candidate-5-1-tjjkwl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26d616a1d020\u003e: \n    failed to create NodePool np-tjjkwl: failed waiting for nodepool=\"np-tjjkwl\" for cluster \"cluster-candidate-5-1-tjjkwl\" in resourcegroup=\"rg-candidate-5-1-q8twtdbssn2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c8cd836c-e28b-40d3-9309-d2867dbc70cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c8cd836c-e28b-40d3-9309-d2867dbc70cf\",\n      \"name\": \"c8cd836c-e28b-40d3-9309-d2867dbc70cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:43:44.796892922Z\",\n      \"endTime\": \"2026-10-08T01:12:45.932063897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-558fx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-q8twtdbssn2z-candidate-tjjkwl--managed/providers/Microsoft.Compute/virtualMachines/t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-558fx to exist\\nMachine t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-7b9jx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-q8twtdbssn2z-candidate-tjjkwl--managed/providers/Microsoft.Compute/virtualMachines/t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-7b9jx to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-558fx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-q8twtdbssn2z-candidate-tjjkwl--managed/providers/Microsoft.Compute/virtualMachines/t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-558fx to exist\\nMachine t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-7b9jx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-q8twtdbssn2z-candidate-tjjkwl--managed/providers/Microsoft.Compute/virtualMachines/t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-7b9jx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tjjkwl\" for cluster \"cluster-candidate-5-1-tjjkwl\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26d616a1d020\u003e: \n    failed to create NodePool np-tjjkwl: failed waiting for nodepool=\"np-tjjkwl\" for cluster \"cluster-candidate-5-1-tjjkwl\" in resourcegroup=\"rg-candidate-5-1-q8twtdbssn2z\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c8cd836c-e28b-40d3-9309-d2867dbc70cf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c8cd836c-e28b-40d3-9309-d2867dbc70cf\",\n      \"name\": \"c8cd836c-e28b-40d3-9309-d2867dbc70cf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:43:44.796892922Z\",\n      \"endTime\": \"2026-10-08T01:12:45.932063897Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-558fx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-q8twtdbssn2z-candidate-tjjkwl--managed/providers/Microsoft.Compute/virtualMachines/t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-558fx to exist\\nMachine t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-7b9jx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-q8twtdbssn2z-candidate-tjjkwl--managed/providers/Microsoft.Compute/virtualMachines/t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-7b9jx to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-558fx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-q8twtdbssn2z-candidate-tjjkwl--managed/providers/Microsoft.Compute/virtualMachines/t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-558fx to exist\\nMachine t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-7b9jx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-q8twtdbssn2z-candidate-tjjkwl--managed/providers/Microsoft.Compute/virtualMachines/t0e4z7t7v6o7i1m-np-tjjkwl-bvg4p-7b9jx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8bd52f1b6a1dfa623432288128879f2add414a992d2cf9938dae9f8637d8d8c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"aaecb179fa0c8619e61820f2147630fb08f8bc80cc1c2f7237571a27bd0b49f3","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30a375627b20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x30a375627ae0\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x30a375627a80\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:296]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30a375627b20\u003e: \n    VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\n    {\n        msg: \"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\",\n        err: \u003c*fmt.wrapError | 0x30a375627ae0\u003e{\n            msg: \"failed to list pods in namespace pullsecret-image-test: client rate limiter Wait returned an error: context deadline exceeded\",\n            err: \u003c*fmt.wrapError | 0x30a375627a80\u003e{\n                msg: \"client rate limiter Wait returned an error: context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"8ea41937321d62b0a4d546bbf42b4298fb3ea9d89af262e908b94ffd6eb83c80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"c8c0700c36a2dd608f6a26fbbe6a7b358648287bea29f72e02d30d5eb97939a8","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1e936bd76030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-vttzcxq26rj2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f55dcf67-ecfc-43cf-8a1f-da9dc6f83710\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f55dcf67-ecfc-43cf-8a1f-da9dc6f83710\",\n      \"name\": \"f55dcf67-ecfc-43cf-8a1f-da9dc6f83710\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:10:25.217518948Z\",\n      \"endTime\": \"2026-10-08T00:39:31.060799155Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-w7d69-bsjvv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-w7d69-fjx74: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-vttzcxq26rj2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-w7d69-fjx74 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-w7d69-bsjvv: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-w7d69-fjx74: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-vttzcxq26rj2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-w7d69-fjx74 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1e936bd76030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-vttzcxq26rj2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f55dcf67-ecfc-43cf-8a1f-da9dc6f83710\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/f55dcf67-ecfc-43cf-8a1f-da9dc6f83710\",\n      \"name\": \"f55dcf67-ecfc-43cf-8a1f-da9dc6f83710\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:10:25.217518948Z\",\n      \"endTime\": \"2026-10-08T00:39:31.060799155Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-w7d69-bsjvv: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-w7d69-fjx74: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-vttzcxq26rj2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-w7d69-fjx74 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-w7d69-bsjvv: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-w7d69-fjx74: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-vttzcxq26rj2--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-w7d69-fjx74 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e494ab871785f5a97792b29f166d05920451682976da5e3bbe878f90ed4562d9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107977652647235584","occurred_at":"2026-10-07T23:33:31Z","failed_at":"e2e","signature_id":"1b1d10ece3b90ada6d195629824ea959318da72c37d7910b78e8868c3143e501","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x99270993880\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:388]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x99270993880\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    {\n        s: \"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6414ac0ffcaccf7efef39a38c0b45dd9d34701a344992b34e8e112eb6e3a27b7","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","6414ac0ffcaccf7efef39a38c0b45dd9d34701a344992b34e8e112eb6e3a27b7","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-07T23:00:37Z","started_at":"2026-10-07T23:00:37Z","completed_at":"2026-10-08T01:26:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-08T01:31:17.962851411Z","timing_metadata_checked_at":"2026-10-08T01:31:17.962851411Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-08T01:31:17.9662271Z","region_metadata_checked_at":"2026-10-08T01:31:17.9662271Z"},"failed_at":["e2e"],"failed_test_count":25,"occurrences":[{"row_id":"07fa7385ef3a0e0db72e0b2ba992c62beaf632f98cb7e152a5dadb9daa565aea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"f7d1fcc30d377fd476b9f57741967c7af1149a2007cd215bbdac2b9536cded49","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nwbn42\" for cluster \"cluster-nightly-4-23-nwbn42\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x57354bbc760\u003e: \n    failed to create NodePool np-nwbn42: failed waiting for nodepool=\"np-nwbn42\" for cluster \"cluster-nightly-4-23-nwbn42\" in resourcegroup=\"rg-nightly-4-23-z9bjz8bcs4gq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67960c4d-8863-47f5-a18c-ab41c5dd77d6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67960c4d-8863-47f5-a18c-ab41c5dd77d6\",\n      \"name\": \"67960c4d-8863-47f5-a18c-ab41c5dd77d6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:50.675391535Z\",\n      \"endTime\": \"2026-10-08T00:12:57.231820143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-5sd9g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-z9bjz8bcs4gq-nightly-nwbn42--managed/providers/Microsoft.Compute/virtualMachines/c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-5sd9g to exist\\nMachine c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-gqc4s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-z9bjz8bcs4gq-nightly-nwbn42--managed/providers/Microsoft.Compute/virtualMachines/c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-gqc4s to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-5sd9g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-z9bjz8bcs4gq-nightly-nwbn42--managed/providers/Microsoft.Compute/virtualMachines/c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-5sd9g to exist\\nMachine c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-gqc4s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-z9bjz8bcs4gq-nightly-nwbn42--managed/providers/Microsoft.Compute/virtualMachines/c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-gqc4s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nwbn42\" for cluster \"cluster-nightly-4-23-nwbn42\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x57354bbc760\u003e: \n    failed to create NodePool np-nwbn42: failed waiting for nodepool=\"np-nwbn42\" for cluster \"cluster-nightly-4-23-nwbn42\" in resourcegroup=\"rg-nightly-4-23-z9bjz8bcs4gq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67960c4d-8863-47f5-a18c-ab41c5dd77d6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67960c4d-8863-47f5-a18c-ab41c5dd77d6\",\n      \"name\": \"67960c4d-8863-47f5-a18c-ab41c5dd77d6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:50.675391535Z\",\n      \"endTime\": \"2026-10-08T00:12:57.231820143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-5sd9g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-z9bjz8bcs4gq-nightly-nwbn42--managed/providers/Microsoft.Compute/virtualMachines/c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-5sd9g to exist\\nMachine c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-gqc4s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-z9bjz8bcs4gq-nightly-nwbn42--managed/providers/Microsoft.Compute/virtualMachines/c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-gqc4s to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-5sd9g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-z9bjz8bcs4gq-nightly-nwbn42--managed/providers/Microsoft.Compute/virtualMachines/c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-5sd9g to exist\\nMachine c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-gqc4s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-23-z9bjz8bcs4gq-nightly-nwbn42--managed/providers/Microsoft.Compute/virtualMachines/c4l7y3i6r1j8y8t-np-nwbn42-zmxvs-gqc4s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1e344828f2da4498ef7893c34665b1cd2d843b72042c94df8fff073128402772","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"2f0ca01949f4ede477fc7bb9d9a950ce013da9ddd6123baf58a79a544d4837fd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-hp5brb\" for cluster \"cluster-nightly-5-1-hp5brb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x215296d8f600\u003e: \n    failed to create NodePool np-hp5brb: failed waiting for nodepool=\"np-hp5brb\" for cluster \"cluster-nightly-5-1-hp5brb\" in resourcegroup=\"rg-nightly-5-1-wnq2x6mnbbng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8919e250-0819-41a0-9b76-ed25de5aa0f7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8919e250-0819-41a0-9b76-ed25de5aa0f7\",\n      \"name\": \"8919e250-0819-41a0-9b76-ed25de5aa0f7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:39.552205268Z\",\n      \"endTime\": \"2026-10-08T00:12:47.433037977Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-9dxv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-wnq2x6mnbbng-nightly-hp5brb--managed/providers/Microsoft.Compute/virtualMachines/t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-9dxv2 to exist\\nMachine t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-h2vjl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-wnq2x6mnbbng-nightly-hp5brb--managed/providers/Microsoft.Compute/virtualMachines/t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-h2vjl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-9dxv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-wnq2x6mnbbng-nightly-hp5brb--managed/providers/Microsoft.Compute/virtualMachines/t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-9dxv2 to exist\\nMachine t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-h2vjl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-wnq2x6mnbbng-nightly-hp5brb--managed/providers/Microsoft.Compute/virtualMachines/t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-h2vjl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-hp5brb\" for cluster \"cluster-nightly-5-1-hp5brb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x215296d8f600\u003e: \n    failed to create NodePool np-hp5brb: failed waiting for nodepool=\"np-hp5brb\" for cluster \"cluster-nightly-5-1-hp5brb\" in resourcegroup=\"rg-nightly-5-1-wnq2x6mnbbng\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8919e250-0819-41a0-9b76-ed25de5aa0f7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8919e250-0819-41a0-9b76-ed25de5aa0f7\",\n      \"name\": \"8919e250-0819-41a0-9b76-ed25de5aa0f7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:39.552205268Z\",\n      \"endTime\": \"2026-10-08T00:12:47.433037977Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-9dxv2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-wnq2x6mnbbng-nightly-hp5brb--managed/providers/Microsoft.Compute/virtualMachines/t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-9dxv2 to exist\\nMachine t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-h2vjl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-wnq2x6mnbbng-nightly-hp5brb--managed/providers/Microsoft.Compute/virtualMachines/t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-h2vjl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-9dxv2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-wnq2x6mnbbng-nightly-hp5brb--managed/providers/Microsoft.Compute/virtualMachines/t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-9dxv2 to exist\\nMachine t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-h2vjl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-1-wnq2x6mnbbng-nightly-hp5brb--managed/providers/Microsoft.Compute/virtualMachines/t4s7o0z3w5g1u6t-np-hp5brb-hrqxs-h2vjl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2127418e83e946222f6c017e713aa378b1e07d7db8cedf483fbff3a2065de0ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"de05f0b33c084620030e8c696e91ec71531528875543bab2b0545455a55994aa","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x964c4f74aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-h5xkjmp44fc8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09493ff9-94b2-4569-b1ee-760501a8111a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09493ff9-94b2-4569-b1ee-760501a8111a\",\n      \"name\": \"09493ff9-94b2-4569-b1ee-760501a8111a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:14:22.696850288Z\",\n      \"endTime\": \"2026-10-07T23:43:29.33911056Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f8s1r9i4d5k7b5r-np-1-p8nkw-7hjqk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-h5xkjmp44fc8--managed/providers/Microsoft.Compute/virtualMachines/f8s1r9i4d5k7b5r-np-1-p8nkw-7hjqk to exist\\nMachine f8s1r9i4d5k7b5r-np-1-p8nkw-mwv7x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-h5xkjmp44fc8--managed/providers/Microsoft.Compute/virtualMachines/f8s1r9i4d5k7b5r-np-1-p8nkw-mwv7x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f8s1r9i4d5k7b5r-np-1-p8nkw-7hjqk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-h5xkjmp44fc8--managed/providers/Microsoft.Compute/virtualMachines/f8s1r9i4d5k7b5r-np-1-p8nkw-7hjqk to exist\\nMachine f8s1r9i4d5k7b5r-np-1-p8nkw-mwv7x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-h5xkjmp44fc8--managed/providers/Microsoft.Compute/virtualMachines/f8s1r9i4d5k7b5r-np-1-p8nkw-mwv7x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x964c4f74aa0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-h5xkjmp44fc8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09493ff9-94b2-4569-b1ee-760501a8111a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09493ff9-94b2-4569-b1ee-760501a8111a\",\n      \"name\": \"09493ff9-94b2-4569-b1ee-760501a8111a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:14:22.696850288Z\",\n      \"endTime\": \"2026-10-07T23:43:29.33911056Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine f8s1r9i4d5k7b5r-np-1-p8nkw-7hjqk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-h5xkjmp44fc8--managed/providers/Microsoft.Compute/virtualMachines/f8s1r9i4d5k7b5r-np-1-p8nkw-7hjqk to exist\\nMachine f8s1r9i4d5k7b5r-np-1-p8nkw-mwv7x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-h5xkjmp44fc8--managed/providers/Microsoft.Compute/virtualMachines/f8s1r9i4d5k7b5r-np-1-p8nkw-mwv7x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine f8s1r9i4d5k7b5r-np-1-p8nkw-7hjqk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-h5xkjmp44fc8--managed/providers/Microsoft.Compute/virtualMachines/f8s1r9i4d5k7b5r-np-1-p8nkw-7hjqk to exist\\nMachine f8s1r9i4d5k7b5r-np-1-p8nkw-mwv7x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-h5xkjmp44fc8--managed/providers/Microsoft.Compute/virtualMachines/f8s1r9i4d5k7b5r-np-1-p8nkw-mwv7x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"245f197640973127f3cf496a7607ca938e028fc7c33d4ef5cc33e481f77f7c6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"140752297cfc5196913f69b2f55f75602e169015ac1cf424a20aa497281b774b","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fa024921260\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-8hhqrj\" resourcegroup=\"rg-np-dg-minor-8hhqrj-7f8bnzmkm895\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-8hhqrj-7f8bnzmkm895/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-8hhqrj/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1fa024921260\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-8hhqrj\" resourcegroup=\"rg-np-dg-minor-8hhqrj-7f8bnzmkm895\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-8hhqrj-7f8bnzmkm895/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-8hhqrj/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2f92bcf0c91d8832c50d84188a7fa9d9ca0f09cbe4ffbaf78892d7f119da23c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"43468b889804001bbbe14bf435f20f53ae868359b8d38f9fb0a8ca86f38b3a14","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e46f700980\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-tdmf8r24g52m\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-tdmf8r24g52m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e46f700980\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-tdmf8r24g52m\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-tdmf8r24g52m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"30ea6ef2e6a46be5672df0b7f896ac6ab776db4b1127c12e73ef650aa4e1409c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"84b2c7e168966ee41c9321960df1699f3c7e6ff0bbf13ad9815db6fc66c666f8","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32c6fec798e0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-vbjww8\" in resourcegroup=\"rg-np-skip-minor-vbjww8-9dlhkbhpd9hb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-vbjww8-9dlhkbhpd9hb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-vbjww8/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x32c6fec798e0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-vbjww8\" in resourcegroup=\"rg-np-skip-minor-vbjww8-9dlhkbhpd9hb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-vbjww8-9dlhkbhpd9hb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-vbjww8/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"372f223c9704e4bd73d4e7e72c79277abbad885167a4c96c0e7e0535e7d12f06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"4fbf30b74782eb5ddefc289814bdb46d277c794e317bb41e0d934ac244ef4a6d","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x269719739890\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cfvg2swczz5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x269719739890\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cfvg2swczz5c/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3863d20528e50c8d0bcf436658e41e2d436fdbb1bf7719c2984b67a5d21083eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"4894e3e856cbc512e113a81c6782b48eb1a23170d9613bd7146b82b50a5ba5be","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2661dd4d00c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-z8pk7tmfp9jj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2661dd4d00c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-z8pk7tmfp9jj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4513cd5cc9d3d2fe4a032c8689597b2f9d8b91c2e76f3409364cd55178e847fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"d4720f51184f2ea80269323a34de851f7131f4cdc47aa604ea02b7639bfb5d5a","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: failed to verify HCP cluster viability for patch-name test\nUnexpected error:\n    \u003c*errors.joinError | 0x2e5cf51ae018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2e5cf5120ca0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf5120c80\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4986960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2e5cf52a2040\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf52a2020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4b086e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:98]: failed to verify HCP cluster viability for patch-name test\nUnexpected error:\n    \u003c*errors.joinError | 0x2e5cf51ae018\u003e: \n    VerifyBasicAccess failed: failed to list services: Unauthorized\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x2e5cf5120ca0\u003e{\n                msg: \"VerifyBasicAccess failed: failed to list services: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf5120c80\u003e{\n                    msg: \"failed to list services: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4986960\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n            \u003c*fmt.wrapError | 0x2e5cf52a2040\u003e{\n                msg: \"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\",\n                err: \u003c*fmt.wrapError | 0x2e5cf52a2020\u003e{\n                    msg: \"failed to list all APIServices: Unauthorized\",\n                    err: \u003c*errors.StatusError | 0x2e5cf4b086e0\u003e{\n                        ErrStatus: {\n                            TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                            ListMeta: {\n                                SelfLink: \"\",\n                                ResourceVersion: \"\",\n                                Continue: \"\",\n                                RemainingItemCount: nil,\n                                ShardInfo: nil,\n                            },\n                            Status: \"Failure\",\n                            Message: \"Unauthorized\",\n                            Reason: \"Unauthorized\",\n                            Details: nil,\n                            Code: 401,\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized"}},{"row_id":"4625adbb398d7452c338545d9c59222faebbd4eb27f8c58d9185d4edd09beb15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"68d4ce843ab001efd587a8e4a37b0a8261c4ad82e127c6e95cfebd9d1f8cff44","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x26d333bbc300\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-ckbnsnb4ttgr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0x26d333bbc300\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-ckbnsnb4ttgr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6895fd88c6c704fbeeb00a63d16e0ca6cb256495d38abbe0180941d375f691cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"764d2f0ddbacda8f6566b981d59dc30f4bcdb0c97dd732f715407bf107cec2b5","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x211d1b817380\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gwwz2wxhg86q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x211d1b817380\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gwwz2wxhg86q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7491fff6ecb689631126801b86a78e3a4ef65ba12eeba474ba65a3a63f175e64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"b614f8950b85c4c4d2722a409ef292ea389bae31115c3e2b81722d59a537583b","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x6c1c704d590\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6dmndbpnqjgk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0x6c1c704d590\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-6dmndbpnqjgk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8201ab5647497ad8c6fd84451167ddc61a0582626f03350ccf0268f777cd0534","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"1ebec80e3d431198968e72632366b202903ff8ad592e0ab7fed494102db1b4fb","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-dl424r\" to 4.23.0-0.nightly-multi-2026-10-06-030402\nUnexpected error:\n    \u003c*exported.ResponseError | 0x23e2acc0cbd0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-kjs4dvf659k9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-dl424r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-dl424r\" to 4.23.0-0.nightly-multi-2026-10-06-030402\nUnexpected error:\n    \u003c*exported.ResponseError | 0x23e2acc0cbd0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-kjs4dvf659k9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-dl424r\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8a8938201048b57150bb9d61fa7032ae01d625ef549c45defbeb98bd1f9fa782","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"5f39b424b8f2fd154227943af3c6cc49910c69a6a961a7b0c48ca3a834eee8a4","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35088dfe27a0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-dwmb74mwp62t\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-dwmb74mwp62t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0x35088dfe27a0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-dwmb74mwp62t\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-dwmb74mwp62t/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8bcec44519d59146e5a6ba76644e2cdd028d2793214fc6d89cc7fd59c5b84f4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"6a259fc11a58d55268cdd102b80e33c281baa34d38f39e72cecc21167dfcf5fb","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-wxpqqn\" to 5.0.0-0.nightly-multi-2026-10-06-215158\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3b69cbc993e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-j89cmzvffx9q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-wxpqqn\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-wxpqqn\" to 5.0.0-0.nightly-multi-2026-10-06-215158\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3b69cbc993e0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-j89cmzvffx9q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-wxpqqn\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8edf0348d423610f306bdf973b98a6d956a690d6e722587d1215ca1d2d57f87b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"1b24e8be23bf1c4a3a07ea52d5db1e9347b68ceaa63a98aa205db1be1d9e27db","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pz47xj\" for cluster \"cluster-nightly-5-0-pz47xj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e380aa507c0\u003e: \n    failed to create NodePool np-pz47xj: failed waiting for nodepool=\"np-pz47xj\" for cluster \"cluster-nightly-5-0-pz47xj\" in resourcegroup=\"rg-nightly-5-0-gv8qpwr567vh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90c14dda-0abc-465a-a86c-5836bdbb53b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90c14dda-0abc-465a-a86c-5836bdbb53b3\",\n      \"name\": \"90c14dda-0abc-465a-a86c-5836bdbb53b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:32.739782591Z\",\n      \"endTime\": \"2026-10-08T00:12:37.114886242Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y2y8x5z5q7w5h6q-np-pz47xj-2xd77-9rb2z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-gv8qpwr567vh-nightly-pz47xj--managed/providers/Microsoft.Compute/virtualMachines/y2y8x5z5q7w5h6q-np-pz47xj-2xd77-9rb2z to exist\\nMachine y2y8x5z5q7w5h6q-np-pz47xj-2xd77-dcwqz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-gv8qpwr567vh-nightly-pz47xj--managed/providers/Microsoft.Compute/virtualMachines/y2y8x5z5q7w5h6q-np-pz47xj-2xd77-dcwqz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y2y8x5z5q7w5h6q-np-pz47xj-2xd77-9rb2z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-gv8qpwr567vh-nightly-pz47xj--managed/providers/Microsoft.Compute/virtualMachines/y2y8x5z5q7w5h6q-np-pz47xj-2xd77-9rb2z to exist\\nMachine y2y8x5z5q7w5h6q-np-pz47xj-2xd77-dcwqz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-gv8qpwr567vh-nightly-pz47xj--managed/providers/Microsoft.Compute/virtualMachines/y2y8x5z5q7w5h6q-np-pz47xj-2xd77-dcwqz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pz47xj\" for cluster \"cluster-nightly-5-0-pz47xj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1e380aa507c0\u003e: \n    failed to create NodePool np-pz47xj: failed waiting for nodepool=\"np-pz47xj\" for cluster \"cluster-nightly-5-0-pz47xj\" in resourcegroup=\"rg-nightly-5-0-gv8qpwr567vh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90c14dda-0abc-465a-a86c-5836bdbb53b3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90c14dda-0abc-465a-a86c-5836bdbb53b3\",\n      \"name\": \"90c14dda-0abc-465a-a86c-5836bdbb53b3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:43:32.739782591Z\",\n      \"endTime\": \"2026-10-08T00:12:37.114886242Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine y2y8x5z5q7w5h6q-np-pz47xj-2xd77-9rb2z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-gv8qpwr567vh-nightly-pz47xj--managed/providers/Microsoft.Compute/virtualMachines/y2y8x5z5q7w5h6q-np-pz47xj-2xd77-9rb2z to exist\\nMachine y2y8x5z5q7w5h6q-np-pz47xj-2xd77-dcwqz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-gv8qpwr567vh-nightly-pz47xj--managed/providers/Microsoft.Compute/virtualMachines/y2y8x5z5q7w5h6q-np-pz47xj-2xd77-dcwqz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine y2y8x5z5q7w5h6q-np-pz47xj-2xd77-9rb2z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-gv8qpwr567vh-nightly-pz47xj--managed/providers/Microsoft.Compute/virtualMachines/y2y8x5z5q7w5h6q-np-pz47xj-2xd77-9rb2z to exist\\nMachine y2y8x5z5q7w5h6q-np-pz47xj-2xd77-dcwqz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-gv8qpwr567vh-nightly-pz47xj--managed/providers/Microsoft.Compute/virtualMachines/y2y8x5z5q7w5h6q-np-pz47xj-2xd77-dcwqz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"94797dc7282e9969f45e58e9c4e9dce5b2a37d9a652d04be2fc735039f150e1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"0f0a376071a48cbdd5611a1be04be8ee77bb833d681538cdad43fbb56b08abe9","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p2kwgg\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n    \u003c*exported.ResponseError | 0xd94e927e240\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-xqwwvdsh9nb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p2kwgg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-p2kwgg\" to 4.21.0-0.nightly-multi-2026-10-02-115050\nUnexpected error:\n    \u003c*exported.ResponseError | 0xd94e927e240\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-xqwwvdsh9nb6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-p2kwgg\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"99752a3edb381e611d323fb2e98acbdcb9ae187ba047b7ec4786a61edd104716","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"f9758d5d36fc27455b6a21116c9b88ebed4012752a67056213ed40b7fb482538","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-nfn8dt\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n    \u003c*exported.ResponseError | 0x353129c55a10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-n89cb96vz49d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-nfn8dt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-nfn8dt\" to 5.1.0-0.nightly-multi-2026-09-30-181926\nUnexpected error:\n    \u003c*exported.ResponseError | 0x353129c55a10\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-n89cb96vz49d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-nfn8dt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"998e126e66d5c47d1d70624a11bb68dae8e74a33646b45742274d44b3fe6ba1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"b15f13ff28e10dd36674e727784b787d32b868277a3a5254568d42d9dddd74c8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bmc2d6\" for cluster \"cluster-nightly-4-22-bmc2d6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe837a83b40\u003e: \n    failed to create NodePool np-bmc2d6: failed waiting for nodepool=\"np-bmc2d6\" for cluster \"cluster-nightly-4-22-bmc2d6\" in resourcegroup=\"rg-nightly-4-22-fbkwmv47s96h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/902c112a-a956-49a6-9b25-1aac16e5506a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/902c112a-a956-49a6-9b25-1aac16e5506a\",\n      \"name\": \"902c112a-a956-49a6-9b25-1aac16e5506a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:39:59.543199325Z\",\n      \"endTime\": \"2026-10-08T00:09:07.262801862Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-4mrgt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-fbkwmv47s96h-nightly-bmc2d6--managed/providers/Microsoft.Compute/virtualMachines/b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-4mrgt to exist\\nMachine b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-cfqsl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-fbkwmv47s96h-nightly-bmc2d6--managed/providers/Microsoft.Compute/virtualMachines/b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-cfqsl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-4mrgt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-fbkwmv47s96h-nightly-bmc2d6--managed/providers/Microsoft.Compute/virtualMachines/b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-4mrgt to exist\\nMachine b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-cfqsl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-fbkwmv47s96h-nightly-bmc2d6--managed/providers/Microsoft.Compute/virtualMachines/b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-cfqsl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bmc2d6\" for cluster \"cluster-nightly-4-22-bmc2d6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe837a83b40\u003e: \n    failed to create NodePool np-bmc2d6: failed waiting for nodepool=\"np-bmc2d6\" for cluster \"cluster-nightly-4-22-bmc2d6\" in resourcegroup=\"rg-nightly-4-22-fbkwmv47s96h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/902c112a-a956-49a6-9b25-1aac16e5506a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/902c112a-a956-49a6-9b25-1aac16e5506a\",\n      \"name\": \"902c112a-a956-49a6-9b25-1aac16e5506a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:39:59.543199325Z\",\n      \"endTime\": \"2026-10-08T00:09:07.262801862Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-4mrgt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-fbkwmv47s96h-nightly-bmc2d6--managed/providers/Microsoft.Compute/virtualMachines/b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-4mrgt to exist\\nMachine b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-cfqsl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-fbkwmv47s96h-nightly-bmc2d6--managed/providers/Microsoft.Compute/virtualMachines/b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-cfqsl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-4mrgt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-fbkwmv47s96h-nightly-bmc2d6--managed/providers/Microsoft.Compute/virtualMachines/b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-4mrgt to exist\\nMachine b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-cfqsl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-22-fbkwmv47s96h-nightly-bmc2d6--managed/providers/Microsoft.Compute/virtualMachines/b3u9y3i7u2b2x2p-np-bmc2d6-2h4tr-cfqsl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9dbe5fdad7869505db007f8894f454e7a209f53ee53110dc680fb8a87318c908","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"cea78ce35fc90771f705f79a4927fdf38597e343e521b69fc7526b14eb7ab849","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1fd50d650030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-chkfwltmpvqk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a08d520-d555-4d1f-b6ab-c391b10d344e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a08d520-d555-4d1f-b6ab-c391b10d344e\",\n      \"name\": \"5a08d520-d555-4d1f-b6ab-c391b10d344e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:14:17.313989392Z\",\n      \"endTime\": \"2026-10-07T23:43:19.479231434Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-xc7p4-6l5l8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-chkfwltmpvqk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-xc7p4-6l5l8 to exist\\nMachine cilium-cluster-cilium-np-xc7p4-pxtxg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-chkfwltmpvqk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-xc7p4-pxtxg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-xc7p4-6l5l8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-chkfwltmpvqk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-xc7p4-6l5l8 to exist\\nMachine cilium-cluster-cilium-np-xc7p4-pxtxg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-chkfwltmpvqk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-xc7p4-pxtxg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x1fd50d650030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-chkfwltmpvqk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a08d520-d555-4d1f-b6ab-c391b10d344e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a08d520-d555-4d1f-b6ab-c391b10d344e\",\n      \"name\": \"5a08d520-d555-4d1f-b6ab-c391b10d344e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:14:17.313989392Z\",\n      \"endTime\": \"2026-10-07T23:43:19.479231434Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-xc7p4-6l5l8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-chkfwltmpvqk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-xc7p4-6l5l8 to exist\\nMachine cilium-cluster-cilium-np-xc7p4-pxtxg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-chkfwltmpvqk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-xc7p4-pxtxg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-xc7p4-6l5l8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-chkfwltmpvqk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-xc7p4-6l5l8 to exist\\nMachine cilium-cluster-cilium-np-xc7p4-pxtxg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-chkfwltmpvqk--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-xc7p4-pxtxg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c87112b6cf774dc1d85caa188c9213f3274f81e1658203c094f0107ee78d18a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"5ad64e3bdfb8fb37ebf569989c2484a58f8d9a29ef2b587e78f90bbd9aca0db6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-55x2r2\" for cluster \"cluster-nightly-4-21-55x2r2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4ebb82b1460\u003e: \n    failed to create NodePool np-55x2r2: failed waiting for nodepool=\"np-55x2r2\" for cluster \"cluster-nightly-4-21-55x2r2\" in resourcegroup=\"rg-nightly-4-21-frqv2hgwnkrq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f248ec39-1150-437e-a025-eddf5336b1bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f248ec39-1150-437e-a025-eddf5336b1bb\",\n      \"name\": \"f248ec39-1150-437e-a025-eddf5336b1bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:39:09.674842789Z\",\n      \"endTime\": \"2026-10-08T00:08:17.294121295Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v3i7l4w7q0g0q1u-np-55x2r2-gggk4-2h4js: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-frqv2hgwnkrq-nightly-55x2r2--managed/providers/Microsoft.Compute/virtualMachines/v3i7l4w7q0g0q1u-np-55x2r2-gggk4-2h4js to exist\\nMachine v3i7l4w7q0g0q1u-np-55x2r2-gggk4-xd7fs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-frqv2hgwnkrq-nightly-55x2r2--managed/providers/Microsoft.Compute/virtualMachines/v3i7l4w7q0g0q1u-np-55x2r2-gggk4-xd7fs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v3i7l4w7q0g0q1u-np-55x2r2-gggk4-2h4js: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-frqv2hgwnkrq-nightly-55x2r2--managed/providers/Microsoft.Compute/virtualMachines/v3i7l4w7q0g0q1u-np-55x2r2-gggk4-2h4js to exist\\nMachine v3i7l4w7q0g0q1u-np-55x2r2-gggk4-xd7fs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-frqv2hgwnkrq-nightly-55x2r2--managed/providers/Microsoft.Compute/virtualMachines/v3i7l4w7q0g0q1u-np-55x2r2-gggk4-xd7fs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-55x2r2\" for cluster \"cluster-nightly-4-21-55x2r2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x4ebb82b1460\u003e: \n    failed to create NodePool np-55x2r2: failed waiting for nodepool=\"np-55x2r2\" for cluster \"cluster-nightly-4-21-55x2r2\" in resourcegroup=\"rg-nightly-4-21-frqv2hgwnkrq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f248ec39-1150-437e-a025-eddf5336b1bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f248ec39-1150-437e-a025-eddf5336b1bb\",\n      \"name\": \"f248ec39-1150-437e-a025-eddf5336b1bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T23:39:09.674842789Z\",\n      \"endTime\": \"2026-10-08T00:08:17.294121295Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine v3i7l4w7q0g0q1u-np-55x2r2-gggk4-2h4js: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-frqv2hgwnkrq-nightly-55x2r2--managed/providers/Microsoft.Compute/virtualMachines/v3i7l4w7q0g0q1u-np-55x2r2-gggk4-2h4js to exist\\nMachine v3i7l4w7q0g0q1u-np-55x2r2-gggk4-xd7fs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-frqv2hgwnkrq-nightly-55x2r2--managed/providers/Microsoft.Compute/virtualMachines/v3i7l4w7q0g0q1u-np-55x2r2-gggk4-xd7fs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine v3i7l4w7q0g0q1u-np-55x2r2-gggk4-2h4js: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-frqv2hgwnkrq-nightly-55x2r2--managed/providers/Microsoft.Compute/virtualMachines/v3i7l4w7q0g0q1u-np-55x2r2-gggk4-2h4js to exist\\nMachine v3i7l4w7q0g0q1u-np-55x2r2-gggk4-xd7fs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-21-frqv2hgwnkrq-nightly-55x2r2--managed/providers/Microsoft.Compute/virtualMachines/v3i7l4w7q0g0q1u-np-55x2r2-gggk4-xd7fs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dc57d9ae57374c73eaff0298c37a4dc601a15014d6e8fcc9a1f6503cc7545043","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"f06515c3d076448409dc5ad1d9d943a2b21906ced720a3fc9685ab778dec06cb","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1c11c2b2b440\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-npbf7r72xkcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\",\n      \"name\": \"26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:01:08.520978007Z\",\n      \"endTime\": \"2026-10-08T00:30:17.207038187Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d to exist\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d to exist\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1c11c2b2b440\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-npbf7r72xkcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\",\n      \"name\": \"26aa56ea-24a6-4dc0-88b5-bb7b84b010bb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-08T00:01:08.520978007Z\",\n      \"endTime\": \"2026-10-08T00:30:17.207038187Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d to exist\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-g9n8d to exist\\nMachine m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-npbf7r72xkcb--managed/providers/Microsoft.Compute/virtualMachines/m1d0t3b5e4r5u1q-np-delete-main-8bjl8-x4tnk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6xhlmlzjgvst\" to finish creating: GET https://management"}},{"row_id":"de15880b994f8b5ffff1a4b204400c5eed0f050a25a0848724e80f4438198102","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"95caab3bc51d32ccb091310bdf33b3e84812faa9fc5c890de7a18f95ac7e465e","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1829881ca3e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7pmxvz\" in resourcegroup=\"rg-np-version-upgrade-7pmxvz-jjjscx2plvvj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7pmxvz-jjjscx2plvvj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7pmxvz/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-10-07-010648\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1829881ca3e0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-7pmxvz\" in resourcegroup=\"rg-np-version-upgrade-7pmxvz-jjjscx2plvvj\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7pmxvz-jjjscx2plvvj/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7pmxvz/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"e74acbf900d4f5013bd2938e548381787781e8dc9e368e9969f2c6c2ee9b57dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"8048cfa91100e6e8b8333a3eeebf145761ee4c1f1cf7dbde205df021632e34c4","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8hxnxn\" to 4.22.0-0.nightly-multi-2026-10-06-184130\nUnexpected error:\n    \u003c*exported.ResponseError | 0x354122bf26f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-jt6zr2jjbm9m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-8hxnxn\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8hxnxn\" to 4.22.0-0.nightly-multi-2026-10-06-184130\nUnexpected error:\n    \u003c*exported.ResponseError | 0x354122bf26f0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-jt6zr2jjbm9m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-8hxnxn\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f8c445155aebcebc440565be5487f8a849be71aac0b9863b8f523ceee2c4d38f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2107969374202630144","occurred_at":"2026-10-07T23:00:37Z","failed_at":"e2e","signature_id":"6037211c53fad4b43e675ddfe0a29c9bf8bd819adc8e5ef742dd4a3d5eac5fbb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x312a48ea2980\u003e: \n    failed to create NodePool np-zgtwtf: failed starting nodepool creation \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\" in resourcegroup=\"rg-nightly-4-20-j42q4dvn7t74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-j42q4dvn7t74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zgtwtf/nodePools/np-zgtwtf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x312a48ea2980\u003e: \n    failed to create NodePool np-zgtwtf: failed starting nodepool creation \"np-zgtwtf\" for cluster \"cluster-nightly-4-20-zgtwtf\" in resourcegroup=\"rg-nightly-4-20-j42q4dvn7t74\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-j42q4dvn7t74/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-zgtwtf/nodePools/np-zgtwtf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-10-07-010648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":25,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5876218c3e485902359b438d646a10e40f4e8e1df2502e7b22dee86c93191389","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc"],"matched_occurrences":25,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-07T19:33:35Z","started_at":"2026-10-07T19:33:35Z","completed_at":"2026-10-07T22:27:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-07T22:33:33.126290931Z","timing_metadata_checked_at":"2026-10-07T22:33:33.126290931Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-07T22:33:33.129234421Z","region_metadata_checked_at":"2026-10-07T22:33:33.129234421Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"31cd6601b653f22e8616d8eed209e2af07a70b4b7d410eba92cdcfa90d2cbf91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"42cc3045165e80f27d068734529151281e6d04adb423b2dda7c90a129047ebf7","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x42d11f950c8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2vrv7v7vlpcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a289f01-1747-41f4-af5e-032e7e217e1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a289f01-1747-41f4-af5e-032e7e217e1d\",\n      \"name\": \"9a289f01-1747-41f4-af5e-032e7e217e1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:08:33.943284812Z\",\n      \"endTime\": \"2026-10-07T20:37:40.405868199Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-lrj9w-6k6fg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-6k6fg to exist\\nMachine cilium-cluster-cilium-np-lrj9w-hx9gl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-hx9gl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-lrj9w-6k6fg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-6k6fg to exist\\nMachine cilium-cluster-cilium-np-lrj9w-hx9gl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-hx9gl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x42d11f950c8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-2vrv7v7vlpcb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a289f01-1747-41f4-af5e-032e7e217e1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/9a289f01-1747-41f4-af5e-032e7e217e1d\",\n      \"name\": \"9a289f01-1747-41f4-af5e-032e7e217e1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:08:33.943284812Z\",\n      \"endTime\": \"2026-10-07T20:37:40.405868199Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-lrj9w-6k6fg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-6k6fg to exist\\nMachine cilium-cluster-cilium-np-lrj9w-hx9gl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-hx9gl to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-lrj9w-6k6fg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-6k6fg to exist\\nMachine cilium-cluster-cilium-np-lrj9w-hx9gl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-2vrv7v7vlpcb--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-lrj9w-hx9gl to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"67156c8b4f767554eddce3d46fa1b949a73e6bd220f458eefa0babde1d453da1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"b80fe52aacb5620760c6bea3b3ed05432ded767100eb2b21c6c1e752e2a3acc1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tng4gr\" for cluster \"cluster-candidate-4-20-tng4gr\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x38fde1c16bc0\u003e: \n    failed to create NodePool np-tng4gr: failed waiting for nodepool=\"np-tng4gr\" for cluster \"cluster-candidate-4-20-tng4gr\" in resourcegroup=\"rg-candidate-4-20-p58t2pzhvwgd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e7e65054-2290-43e7-8239-944663da4cfa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e7e65054-2290-43e7-8239-944663da4cfa\",\n      \"name\": \"e7e65054-2290-43e7-8239-944663da4cfa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:35:56.19805172Z\",\n      \"endTime\": \"2026-10-07T21:04:58.851835237Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-55wrp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-p58t2pzhvwgd-candidate-tng4gr--managed/providers/Microsoft.Compute/virtualMachines/x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-55wrp to exist\\nMachine x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-vlpx2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-p58t2pzhvwgd-candidate-tng4gr--managed/providers/Microsoft.Compute/virtualMachines/x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-vlpx2 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-55wrp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-p58t2pzhvwgd-candidate-tng4gr--managed/providers/Microsoft.Compute/virtualMachines/x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-55wrp to exist\\nMachine x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-vlpx2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-p58t2pzhvwgd-candidate-tng4gr--managed/providers/Microsoft.Compute/virtualMachines/x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-vlpx2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-tng4gr\" for cluster \"cluster-candidate-4-20-tng4gr\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x38fde1c16bc0\u003e: \n    failed to create NodePool np-tng4gr: failed waiting for nodepool=\"np-tng4gr\" for cluster \"cluster-candidate-4-20-tng4gr\" in resourcegroup=\"rg-candidate-4-20-p58t2pzhvwgd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e7e65054-2290-43e7-8239-944663da4cfa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e7e65054-2290-43e7-8239-944663da4cfa\",\n      \"name\": \"e7e65054-2290-43e7-8239-944663da4cfa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:35:56.19805172Z\",\n      \"endTime\": \"2026-10-07T21:04:58.851835237Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-55wrp: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-p58t2pzhvwgd-candidate-tng4gr--managed/providers/Microsoft.Compute/virtualMachines/x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-55wrp to exist\\nMachine x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-vlpx2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-p58t2pzhvwgd-candidate-tng4gr--managed/providers/Microsoft.Compute/virtualMachines/x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-vlpx2 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-55wrp: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-p58t2pzhvwgd-candidate-tng4gr--managed/providers/Microsoft.Compute/virtualMachines/x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-55wrp to exist\\nMachine x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-vlpx2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-p58t2pzhvwgd-candidate-tng4gr--managed/providers/Microsoft.Compute/virtualMachines/x4a6c1x6c1p1p4v-np-tng4gr-mkzdj-vlpx2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ff9b24fd95a3b0fa2813da8d3fd3065f4a1af183a4e3c7809e84e0a2bfc8f61","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"dd2af2d4c434317800a4e02b03119effcc044182d265a3a22bcae48911b84b86","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe2ef1777c20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-jtndxjs7xcxx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8ec092cc-d390-467a-9cfc-6f58b823214c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8ec092cc-d390-467a-9cfc-6f58b823214c\",\n      \"name\": \"8ec092cc-d390-467a-9cfc-6f58b823214c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:08:57.296668233Z\",\n      \"endTime\": \"2026-10-07T20:38:00.537683031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-bnc9s-6hbvx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtndxjs7xcxx--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-bnc9s-6hbvx to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-bnc9s-6hbvx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtndxjs7xcxx--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-bnc9s-6hbvx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xe2ef1777c20\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-jtndxjs7xcxx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8ec092cc-d390-467a-9cfc-6f58b823214c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/8ec092cc-d390-467a-9cfc-6f58b823214c\",\n      \"name\": \"8ec092cc-d390-467a-9cfc-6f58b823214c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:08:57.296668233Z\",\n      \"endTime\": \"2026-10-07T20:38:00.537683031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-bnc9s-6hbvx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtndxjs7xcxx--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-bnc9s-6hbvx to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-bnc9s-6hbvx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-jtndxjs7xcxx--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-bnc9s-6hbvx to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9a21343337fbc3e7c8c8c5480d861ffb9918f9998b2d7def972d9ea7c6f310c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"cba8f12de6a543925775d61a4dc9325c74c4f0c58a30cea6a9fa01d63b96e361","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c37066c9ac0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-p77gk7tcv5q9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c8efd0a6-5b62-424e-9f06-59f6979e8fd6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c8efd0a6-5b62-424e-9f06-59f6979e8fd6\",\n      \"name\": \"c8efd0a6-5b62-424e-9f06-59f6979e8fd6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:07:39.49072968Z\",\n      \"endTime\": \"2026-10-07T20:36:40.369440399Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5l4t9b8y5f2w6p-np-1-l68gq-68sdx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-p77gk7tcv5q9--managed/providers/Microsoft.Compute/virtualMachines/q5l4t9b8y5f2w6p-np-1-l68gq-68sdx to exist\\nMachine q5l4t9b8y5f2w6p-np-1-l68gq-6pp4h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-p77gk7tcv5q9--managed/providers/Microsoft.Compute/virtualMachines/q5l4t9b8y5f2w6p-np-1-l68gq-6pp4h to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5l4t9b8y5f2w6p-np-1-l68gq-68sdx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-p77gk7tcv5q9--managed/providers/Microsoft.Compute/virtualMachines/q5l4t9b8y5f2w6p-np-1-l68gq-68sdx to exist\\nMachine q5l4t9b8y5f2w6p-np-1-l68gq-6pp4h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-p77gk7tcv5q9--managed/providers/Microsoft.Compute/virtualMachines/q5l4t9b8y5f2w6p-np-1-l68gq-6pp4h to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1c37066c9ac0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-p77gk7tcv5q9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c8efd0a6-5b62-424e-9f06-59f6979e8fd6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/c8efd0a6-5b62-424e-9f06-59f6979e8fd6\",\n      \"name\": \"c8efd0a6-5b62-424e-9f06-59f6979e8fd6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:07:39.49072968Z\",\n      \"endTime\": \"2026-10-07T20:36:40.369440399Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5l4t9b8y5f2w6p-np-1-l68gq-68sdx: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-p77gk7tcv5q9--managed/providers/Microsoft.Compute/virtualMachines/q5l4t9b8y5f2w6p-np-1-l68gq-68sdx to exist\\nMachine q5l4t9b8y5f2w6p-np-1-l68gq-6pp4h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-p77gk7tcv5q9--managed/providers/Microsoft.Compute/virtualMachines/q5l4t9b8y5f2w6p-np-1-l68gq-6pp4h to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5l4t9b8y5f2w6p-np-1-l68gq-68sdx: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-p77gk7tcv5q9--managed/providers/Microsoft.Compute/virtualMachines/q5l4t9b8y5f2w6p-np-1-l68gq-68sdx to exist\\nMachine q5l4t9b8y5f2w6p-np-1-l68gq-6pp4h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-p77gk7tcv5q9--managed/providers/Microsoft.Compute/virtualMachines/q5l4t9b8y5f2w6p-np-1-l68gq-6pp4h to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a4bd87c45178ef852bf89bf116c645a30e467d0dbbc0382d318bb3ad77ea8898","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"bb8fa03a2e39c2209193639a0d46220b87e342b6202a3ad16c83d99f1ef37edf","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9cltms\" for cluster \"cluster-candidate-4-21-9cltms\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17226b681ca0\u003e: \n    failed to create NodePool np-9cltms: failed waiting for nodepool=\"np-9cltms\" for cluster \"cluster-candidate-4-21-9cltms\" in resourcegroup=\"rg-candidate-4-21-5wcq5gcwvh45\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5a18c5c3-1285-4432-9e6e-bd738edcf21c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5a18c5c3-1285-4432-9e6e-bd738edcf21c\",\n      \"name\": \"5a18c5c3-1285-4432-9e6e-bd738edcf21c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:36:07.718000912Z\",\n      \"endTime\": \"2026-10-07T21:05:08.775957747Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m2a5h3r9k9e3a3i-np-9cltms-rbspc-5q8n5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5wcq5gcwvh45-candidate-9cltms--managed/providers/Microsoft.Compute/virtualMachines/m2a5h3r9k9e3a3i-np-9cltms-rbspc-5q8n5 to exist\\nMachine m2a5h3r9k9e3a3i-np-9cltms-rbspc-76sz5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5wcq5gcwvh45-candidate-9cltms--managed/providers/Microsoft.Compute/virtualMachines/m2a5h3r9k9e3a3i-np-9cltms-rbspc-76sz5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m2a5h3r9k9e3a3i-np-9cltms-rbspc-5q8n5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5wcq5gcwvh45-candidate-9cltms--managed/providers/Microsoft.Compute/virtualMachines/m2a5h3r9k9e3a3i-np-9cltms-rbspc-5q8n5 to exist\\nMachine m2a5h3r9k9e3a3i-np-9cltms-rbspc-76sz5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5wcq5gcwvh45-candidate-9cltms--managed/providers/Microsoft.Compute/virtualMachines/m2a5h3r9k9e3a3i-np-9cltms-rbspc-76sz5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-9cltms\" for cluster \"cluster-candidate-4-21-9cltms\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17226b681ca0\u003e: \n    failed to create NodePool np-9cltms: failed waiting for nodepool=\"np-9cltms\" for cluster \"cluster-candidate-4-21-9cltms\" in resourcegroup=\"rg-candidate-4-21-5wcq5gcwvh45\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5a18c5c3-1285-4432-9e6e-bd738edcf21c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5a18c5c3-1285-4432-9e6e-bd738edcf21c\",\n      \"name\": \"5a18c5c3-1285-4432-9e6e-bd738edcf21c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:36:07.718000912Z\",\n      \"endTime\": \"2026-10-07T21:05:08.775957747Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m2a5h3r9k9e3a3i-np-9cltms-rbspc-5q8n5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5wcq5gcwvh45-candidate-9cltms--managed/providers/Microsoft.Compute/virtualMachines/m2a5h3r9k9e3a3i-np-9cltms-rbspc-5q8n5 to exist\\nMachine m2a5h3r9k9e3a3i-np-9cltms-rbspc-76sz5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5wcq5gcwvh45-candidate-9cltms--managed/providers/Microsoft.Compute/virtualMachines/m2a5h3r9k9e3a3i-np-9cltms-rbspc-76sz5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m2a5h3r9k9e3a3i-np-9cltms-rbspc-5q8n5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5wcq5gcwvh45-candidate-9cltms--managed/providers/Microsoft.Compute/virtualMachines/m2a5h3r9k9e3a3i-np-9cltms-rbspc-5q8n5 to exist\\nMachine m2a5h3r9k9e3a3i-np-9cltms-rbspc-76sz5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-5wcq5gcwvh45-candidate-9cltms--managed/providers/Microsoft.Compute/virtualMachines/m2a5h3r9k9e3a3i-np-9cltms-rbspc-76sz5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"aeec2c41ac9534ba0448c41015ccbdfcc8ea0dbdfb3a033c0e7ee0ba2c11a9ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"c7f5bdc4ebc923e20233f949e36f2ccb4fee458a453a1ee7a629440470bf56a8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6rlkdh\" for cluster \"cluster-candidate-4-23-6rlkdh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15014d7b5940\u003e: \n    failed to create NodePool np-6rlkdh: failed waiting for nodepool=\"np-6rlkdh\" for cluster \"cluster-candidate-4-23-6rlkdh\" in resourcegroup=\"rg-candidate-4-23-cbh64xfpcppm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30c1fd25-bea7-4668-ba77-9639e04fbe20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30c1fd25-bea7-4668-ba77-9639e04fbe20\",\n      \"name\": \"30c1fd25-bea7-4668-ba77-9639e04fbe20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:41:47.534201418Z\",\n      \"endTime\": \"2026-10-07T21:10:48.799809829Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-tf4n9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cbh64xfpcppm-candidate-6rlkdh--managed/providers/Microsoft.Compute/virtualMachines/o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-tf4n9 to exist\\nMachine o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-zfbl8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cbh64xfpcppm-candidate-6rlkdh--managed/providers/Microsoft.Compute/virtualMachines/o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-zfbl8 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-tf4n9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cbh64xfpcppm-candidate-6rlkdh--managed/providers/Microsoft.Compute/virtualMachines/o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-tf4n9 to exist\\nMachine o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-zfbl8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cbh64xfpcppm-candidate-6rlkdh--managed/providers/Microsoft.Compute/virtualMachines/o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-zfbl8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6rlkdh\" for cluster \"cluster-candidate-4-23-6rlkdh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15014d7b5940\u003e: \n    failed to create NodePool np-6rlkdh: failed waiting for nodepool=\"np-6rlkdh\" for cluster \"cluster-candidate-4-23-6rlkdh\" in resourcegroup=\"rg-candidate-4-23-cbh64xfpcppm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30c1fd25-bea7-4668-ba77-9639e04fbe20\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/30c1fd25-bea7-4668-ba77-9639e04fbe20\",\n      \"name\": \"30c1fd25-bea7-4668-ba77-9639e04fbe20\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:41:47.534201418Z\",\n      \"endTime\": \"2026-10-07T21:10:48.799809829Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-tf4n9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cbh64xfpcppm-candidate-6rlkdh--managed/providers/Microsoft.Compute/virtualMachines/o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-tf4n9 to exist\\nMachine o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-zfbl8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cbh64xfpcppm-candidate-6rlkdh--managed/providers/Microsoft.Compute/virtualMachines/o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-zfbl8 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-tf4n9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cbh64xfpcppm-candidate-6rlkdh--managed/providers/Microsoft.Compute/virtualMachines/o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-tf4n9 to exist\\nMachine o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-zfbl8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-cbh64xfpcppm-candidate-6rlkdh--managed/providers/Microsoft.Compute/virtualMachines/o2l4f8h5q8g0s9v-np-6rlkdh-k4ds7-zfbl8 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b5ee8730acc3912f88f890f3f35587b03cabcf76858d907f44189c9088e2e38a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"e3fb4a2b49a31e56c0902493d1ea2275a273a2c53bab17b6f360121094551afe","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kddk55\" for cluster \"cluster-candidate-5-1-kddk55\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13bc8950a6a0\u003e: \n    failed to create NodePool np-kddk55: failed waiting for nodepool=\"np-kddk55\" for cluster \"cluster-candidate-5-1-kddk55\" in resourcegroup=\"rg-candidate-5-1-fsc4c2dfddtk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e435f098-79af-4365-b5b1-76c03affaa2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e435f098-79af-4365-b5b1-76c03affaa2b\",\n      \"name\": \"e435f098-79af-4365-b5b1-76c03affaa2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:43:26.129269305Z\",\n      \"endTime\": \"2026-10-07T21:12:28.780512619Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5q7c9q9x4f1p2k-np-kddk55-tvdpc-2w9m8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-fsc4c2dfddtk-candidate-kddk55--managed/providers/Microsoft.Compute/virtualMachines/q5q7c9q9x4f1p2k-np-kddk55-tvdpc-2w9m8 to exist\\nMachine q5q7c9q9x4f1p2k-np-kddk55-tvdpc-h55rn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-fsc4c2dfddtk-candidate-kddk55--managed/providers/Microsoft.Compute/virtualMachines/q5q7c9q9x4f1p2k-np-kddk55-tvdpc-h55rn to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5q7c9q9x4f1p2k-np-kddk55-tvdpc-2w9m8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-fsc4c2dfddtk-candidate-kddk55--managed/providers/Microsoft.Compute/virtualMachines/q5q7c9q9x4f1p2k-np-kddk55-tvdpc-2w9m8 to exist\\nMachine q5q7c9q9x4f1p2k-np-kddk55-tvdpc-h55rn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-fsc4c2dfddtk-candidate-kddk55--managed/providers/Microsoft.Compute/virtualMachines/q5q7c9q9x4f1p2k-np-kddk55-tvdpc-h55rn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-kddk55\" for cluster \"cluster-candidate-5-1-kddk55\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13bc8950a6a0\u003e: \n    failed to create NodePool np-kddk55: failed waiting for nodepool=\"np-kddk55\" for cluster \"cluster-candidate-5-1-kddk55\" in resourcegroup=\"rg-candidate-5-1-fsc4c2dfddtk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e435f098-79af-4365-b5b1-76c03affaa2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/e435f098-79af-4365-b5b1-76c03affaa2b\",\n      \"name\": \"e435f098-79af-4365-b5b1-76c03affaa2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:43:26.129269305Z\",\n      \"endTime\": \"2026-10-07T21:12:28.780512619Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q5q7c9q9x4f1p2k-np-kddk55-tvdpc-2w9m8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-fsc4c2dfddtk-candidate-kddk55--managed/providers/Microsoft.Compute/virtualMachines/q5q7c9q9x4f1p2k-np-kddk55-tvdpc-2w9m8 to exist\\nMachine q5q7c9q9x4f1p2k-np-kddk55-tvdpc-h55rn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-fsc4c2dfddtk-candidate-kddk55--managed/providers/Microsoft.Compute/virtualMachines/q5q7c9q9x4f1p2k-np-kddk55-tvdpc-h55rn to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q5q7c9q9x4f1p2k-np-kddk55-tvdpc-2w9m8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-fsc4c2dfddtk-candidate-kddk55--managed/providers/Microsoft.Compute/virtualMachines/q5q7c9q9x4f1p2k-np-kddk55-tvdpc-2w9m8 to exist\\nMachine q5q7c9q9x4f1p2k-np-kddk55-tvdpc-h55rn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-fsc4c2dfddtk-candidate-kddk55--managed/providers/Microsoft.Compute/virtualMachines/q5q7c9q9x4f1p2k-np-kddk55-tvdpc-h55rn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d2070bbd59b024a66b02fe70c353ec4fe821190585c4438e59650eb2ab5aec12","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"19f8dd197db68f57fa6b4eeb4e85cec78926fb3e66586ba3c7b4f6152ea72984","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d0dde8c3780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cdl8czz92fjt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b34011e-02fa-407e-bf8e-697c866145e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b34011e-02fa-407e-bf8e-697c866145e8\",\n      \"name\": \"1b34011e-02fa-407e-bf8e-697c866145e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T21:30:13.060183446Z\",\n      \"endTime\": \"2026-10-07T21:59:13.201730463Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68 to exist\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68 to exist\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d0dde8c3780\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cdl8czz92fjt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b34011e-02fa-407e-bf8e-697c866145e8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b34011e-02fa-407e-bf8e-697c866145e8\",\n      \"name\": \"1b34011e-02fa-407e-bf8e-697c866145e8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T21:30:13.060183446Z\",\n      \"endTime\": \"2026-10-07T21:59:13.201730463Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68 to exist\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-qrr68 to exist\\nMachine u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cdl8czz92fjt--managed/providers/Microsoft.Compute/virtualMachines/u7f6c6k9k0a0v7e-np-deletable-gh9lt-tbkcs to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6xhlmlzjgvst\" to finish creating: GET https://management"}},{"row_id":"e8777de7aee8a162d4bf01e014e30003ff305a310f3b3c04d0134c98214a5c65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"9785d8491224b970db4521504142e02625dbd60d73aaa543ecd4460be888e210","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-97mck8\" for cluster \"cluster-candidate-4-22-97mck8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x229053c35120\u003e: \n    failed to create NodePool np-97mck8: failed waiting for nodepool=\"np-97mck8\" for cluster \"cluster-candidate-4-22-97mck8\" in resourcegroup=\"rg-candidate-4-22-j7f8d9chl5df\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b172bf9-41e9-401d-953d-ee155521aa75\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b172bf9-41e9-401d-953d-ee155521aa75\",\n      \"name\": \"5b172bf9-41e9-401d-953d-ee155521aa75\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:37:15.629028583Z\",\n      \"endTime\": \"2026-10-07T21:06:18.880533991Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8r3n7p4s1u6l5s-np-97mck8-h57hq-b2j6q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j7f8d9chl5df-candidate-97mck8--managed/providers/Microsoft.Compute/virtualMachines/z8r3n7p4s1u6l5s-np-97mck8-h57hq-b2j6q to exist\\nMachine z8r3n7p4s1u6l5s-np-97mck8-h57hq-h6n6d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j7f8d9chl5df-candidate-97mck8--managed/providers/Microsoft.Compute/virtualMachines/z8r3n7p4s1u6l5s-np-97mck8-h57hq-h6n6d to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8r3n7p4s1u6l5s-np-97mck8-h57hq-b2j6q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j7f8d9chl5df-candidate-97mck8--managed/providers/Microsoft.Compute/virtualMachines/z8r3n7p4s1u6l5s-np-97mck8-h57hq-b2j6q to exist\\nMachine z8r3n7p4s1u6l5s-np-97mck8-h57hq-h6n6d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j7f8d9chl5df-candidate-97mck8--managed/providers/Microsoft.Compute/virtualMachines/z8r3n7p4s1u6l5s-np-97mck8-h57hq-h6n6d to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-97mck8\" for cluster \"cluster-candidate-4-22-97mck8\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x229053c35120\u003e: \n    failed to create NodePool np-97mck8: failed waiting for nodepool=\"np-97mck8\" for cluster \"cluster-candidate-4-22-97mck8\" in resourcegroup=\"rg-candidate-4-22-j7f8d9chl5df\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b172bf9-41e9-401d-953d-ee155521aa75\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5b172bf9-41e9-401d-953d-ee155521aa75\",\n      \"name\": \"5b172bf9-41e9-401d-953d-ee155521aa75\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:37:15.629028583Z\",\n      \"endTime\": \"2026-10-07T21:06:18.880533991Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine z8r3n7p4s1u6l5s-np-97mck8-h57hq-b2j6q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j7f8d9chl5df-candidate-97mck8--managed/providers/Microsoft.Compute/virtualMachines/z8r3n7p4s1u6l5s-np-97mck8-h57hq-b2j6q to exist\\nMachine z8r3n7p4s1u6l5s-np-97mck8-h57hq-h6n6d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j7f8d9chl5df-candidate-97mck8--managed/providers/Microsoft.Compute/virtualMachines/z8r3n7p4s1u6l5s-np-97mck8-h57hq-h6n6d to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine z8r3n7p4s1u6l5s-np-97mck8-h57hq-b2j6q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j7f8d9chl5df-candidate-97mck8--managed/providers/Microsoft.Compute/virtualMachines/z8r3n7p4s1u6l5s-np-97mck8-h57hq-b2j6q to exist\\nMachine z8r3n7p4s1u6l5s-np-97mck8-h57hq-h6n6d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-j7f8d9chl5df-candidate-97mck8--managed/providers/Microsoft.Compute/virtualMachines/z8r3n7p4s1u6l5s-np-97mck8-h57hq-h6n6d to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f2410e5ac92f6c5b85a4da91696f3dc0a615f3236fcd380890f1cb3e7dbc351d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"753d143e88447561451de08944a9408e6f40d29e02738dd21a3764fd6cbe715c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x32859edcc030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-bngnhgkwvt2t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ba4b9fd7-7f9c-4a19-89d1-ba7979c31bd3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ba4b9fd7-7f9c-4a19-89d1-ba7979c31bd3\",\n      \"name\": \"ba4b9fd7-7f9c-4a19-89d1-ba7979c31bd3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:11:50.180436689Z\",\n      \"endTime\": \"2026-10-07T20:40:50.55544909Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7x7rd-96k8w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bngnhgkwvt2t--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-7x7rd-96k8w to exist\\nMachine agg-cluster-agg-np-a-7x7rd-bx6ks: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7x7rd-bx6ks: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-7x7rd-96k8w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bngnhgkwvt2t--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-7x7rd-96k8w to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x32859edcc030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-bngnhgkwvt2t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ba4b9fd7-7f9c-4a19-89d1-ba7979c31bd3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/ba4b9fd7-7f9c-4a19-89d1-ba7979c31bd3\",\n      \"name\": \"ba4b9fd7-7f9c-4a19-89d1-ba7979c31bd3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:11:50.180436689Z\",\n      \"endTime\": \"2026-10-07T20:40:50.55544909Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-7x7rd-96k8w: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bngnhgkwvt2t--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-7x7rd-96k8w to exist\\nMachine agg-cluster-agg-np-a-7x7rd-bx6ks: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-7x7rd-bx6ks: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-7x7rd-96k8w: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bngnhgkwvt2t--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-7x7rd-96k8w to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fc65d3088edc398971bfa11ea96e3853598eb2dab6d0d73583e822bd224ee56b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107917271417491456","occurred_at":"2026-10-07T19:33:35Z","failed_at":"e2e","signature_id":"04ad7125fe4905ae031c4007581e188577bf5010611a020b6a2efc132c5dc319","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-42p26n\" for cluster \"cluster-candidate-5-0-42p26n\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a220e39e0\u003e: \n    failed to create NodePool np-42p26n: failed waiting for nodepool=\"np-42p26n\" for cluster \"cluster-candidate-5-0-42p26n\" in resourcegroup=\"rg-candidate-5-0-ggx4tjzkw6bv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4cfad558-f967-4b4f-bc63-df85c869948f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4cfad558-f967-4b4f-bc63-df85c869948f\",\n      \"name\": \"4cfad558-f967-4b4f-bc63-df85c869948f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:41:47.770756531Z\",\n      \"endTime\": \"2026-10-07T21:10:48.852402374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g1e2k3e1t3e6h3w-np-42p26n-gchjq-594cc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-ggx4tjzkw6bv-candidate-42p26n--managed/providers/Microsoft.Compute/virtualMachines/g1e2k3e1t3e6h3w-np-42p26n-gchjq-594cc to exist\\nMachine g1e2k3e1t3e6h3w-np-42p26n-gchjq-f9xb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-ggx4tjzkw6bv-candidate-42p26n--managed/providers/Microsoft.Compute/virtualMachines/g1e2k3e1t3e6h3w-np-42p26n-gchjq-f9xb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g1e2k3e1t3e6h3w-np-42p26n-gchjq-594cc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-ggx4tjzkw6bv-candidate-42p26n--managed/providers/Microsoft.Compute/virtualMachines/g1e2k3e1t3e6h3w-np-42p26n-gchjq-594cc to exist\\nMachine g1e2k3e1t3e6h3w-np-42p26n-gchjq-f9xb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-ggx4tjzkw6bv-candidate-42p26n--managed/providers/Microsoft.Compute/virtualMachines/g1e2k3e1t3e6h3w-np-42p26n-gchjq-f9xb4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-42p26n\" for cluster \"cluster-candidate-5-0-42p26n\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a220e39e0\u003e: \n    failed to create NodePool np-42p26n: failed waiting for nodepool=\"np-42p26n\" for cluster \"cluster-candidate-5-0-42p26n\" in resourcegroup=\"rg-candidate-5-0-ggx4tjzkw6bv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4cfad558-f967-4b4f-bc63-df85c869948f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4cfad558-f967-4b4f-bc63-df85c869948f\",\n      \"name\": \"4cfad558-f967-4b4f-bc63-df85c869948f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T20:41:47.770756531Z\",\n      \"endTime\": \"2026-10-07T21:10:48.852402374Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g1e2k3e1t3e6h3w-np-42p26n-gchjq-594cc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-ggx4tjzkw6bv-candidate-42p26n--managed/providers/Microsoft.Compute/virtualMachines/g1e2k3e1t3e6h3w-np-42p26n-gchjq-594cc to exist\\nMachine g1e2k3e1t3e6h3w-np-42p26n-gchjq-f9xb4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-ggx4tjzkw6bv-candidate-42p26n--managed/providers/Microsoft.Compute/virtualMachines/g1e2k3e1t3e6h3w-np-42p26n-gchjq-f9xb4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g1e2k3e1t3e6h3w-np-42p26n-gchjq-594cc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-ggx4tjzkw6bv-candidate-42p26n--managed/providers/Microsoft.Compute/virtualMachines/g1e2k3e1t3e6h3w-np-42p26n-gchjq-594cc to exist\\nMachine g1e2k3e1t3e6h3w-np-42p26n-gchjq-f9xb4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-ggx4tjzkw6bv-candidate-42p26n--managed/providers/Microsoft.Compute/virtualMachines/g1e2k3e1t3e6h3w-np-42p26n-gchjq-f9xb4 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-07T16:34:17Z","started_at":"2026-10-07T16:34:17Z","completed_at":"2026-10-07T19:28:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-07T19:36:42.177661822Z","timing_metadata_checked_at":"2026-10-07T19:36:42.177661822Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-07T19:36:42.180863712Z","region_metadata_checked_at":"2026-10-07T19:36:42.180863712Z"},"failed_at":["e2e"],"failed_test_count":11,"occurrences":[{"row_id":"3a60818c9932680d23dd2b4087bccdc53bd0bf5e1fa42cb852841a68c3fe0e8f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"29adca326faa1bc8f35a40a7651e6edc98aa900d6b8aa8c789d1b2db2ab468ec","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x35ed21e504a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6xhlmlzjgvst\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/12991b47-b015-4cbd-9985-f7c3a34af217\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/12991b47-b015-4cbd-9985-f7c3a34af217\",\n      \"name\": \"12991b47-b015-4cbd-9985-f7c3a34af217\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:35:02.297058773Z\",\n      \"endTime\": \"2026-10-07T19:04:10.718534768Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n4a0f5c4d7s5h7j-np-deletable-sn575-76h9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6xhlmlzjgvst--managed/providers/Microsoft.Compute/virtualMachines/n4a0f5c4d7s5h7j-np-deletable-sn575-76h9t to exist\\nMachine n4a0f5c4d7s5h7j-np-deletable-sn575-blj2k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6xhlmlzjgvst--managed/providers/Microsoft.Compute/virtualMachines/n4a0f5c4d7s5h7j-np-deletable-sn575-blj2k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n4a0f5c4d7s5h7j-np-deletable-sn575-76h9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6xhlmlzjgvst--managed/providers/Microsoft.Compute/virtualMachines/n4a0f5c4d7s5h7j-np-deletable-sn575-76h9t to exist\\nMachine n4a0f5c4d7s5h7j-np-deletable-sn575-blj2k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6xhlmlzjgvst--managed/providers/Microsoft.Compute/virtualMachines/n4a0f5c4d7s5h7j-np-deletable-sn575-blj2k to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x35ed21e504a0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6xhlmlzjgvst\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/12991b47-b015-4cbd-9985-f7c3a34af217\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/12991b47-b015-4cbd-9985-f7c3a34af217\",\n      \"name\": \"12991b47-b015-4cbd-9985-f7c3a34af217\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T18:35:02.297058773Z\",\n      \"endTime\": \"2026-10-07T19:04:10.718534768Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n4a0f5c4d7s5h7j-np-deletable-sn575-76h9t: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6xhlmlzjgvst--managed/providers/Microsoft.Compute/virtualMachines/n4a0f5c4d7s5h7j-np-deletable-sn575-76h9t to exist\\nMachine n4a0f5c4d7s5h7j-np-deletable-sn575-blj2k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6xhlmlzjgvst--managed/providers/Microsoft.Compute/virtualMachines/n4a0f5c4d7s5h7j-np-deletable-sn575-blj2k to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n4a0f5c4d7s5h7j-np-deletable-sn575-76h9t: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6xhlmlzjgvst--managed/providers/Microsoft.Compute/virtualMachines/n4a0f5c4d7s5h7j-np-deletable-sn575-76h9t to exist\\nMachine n4a0f5c4d7s5h7j-np-deletable-sn575-blj2k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-6xhlmlzjgvst--managed/providers/Microsoft.Compute/virtualMachines/n4a0f5c4d7s5h7j-np-deletable-sn575-blj2k to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6xhlmlzjgvst\" to finish creating: GET https://management"}},{"row_id":"448d0fa4de1b1c2bd0e29ffbfd9f2543b5028b9e05dcc89e5c661d2d4868323f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"fbd0c561558b517a4ab86f8ad9b18ffae8f602f9d4cdc462cbca34d38dfdb848","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lscpsw\" for cluster \"cluster-candidate-5-1-lscpsw\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a793a7485a0\u003e: \n    failed to create NodePool np-lscpsw: failed waiting for nodepool=\"np-lscpsw\" for cluster \"cluster-candidate-5-1-lscpsw\" in resourcegroup=\"rg-candidate-5-1-xrcj42xlnjvs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5813dcb9-1643-4e9b-822d-a0503544e72d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5813dcb9-1643-4e9b-822d-a0503544e72d\",\n      \"name\": \"5813dcb9-1643-4e9b-822d-a0503544e72d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:46:26.081906797Z\",\n      \"endTime\": \"2026-10-07T18:15:34.783886022Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c9q8s0r5e8m4v4l-np-lscpsw-xdz87-b86dg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-xrcj42xlnjvs-candidate-lscpsw--managed/providers/Microsoft.Compute/virtualMachines/c9q8s0r5e8m4v4l-np-lscpsw-xdz87-b86dg to exist\\nMachine c9q8s0r5e8m4v4l-np-lscpsw-xdz87-qxpvk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-xrcj42xlnjvs-candidate-lscpsw--managed/providers/Microsoft.Compute/virtualMachines/c9q8s0r5e8m4v4l-np-lscpsw-xdz87-qxpvk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c9q8s0r5e8m4v4l-np-lscpsw-xdz87-b86dg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-xrcj42xlnjvs-candidate-lscpsw--managed/providers/Microsoft.Compute/virtualMachines/c9q8s0r5e8m4v4l-np-lscpsw-xdz87-b86dg to exist\\nMachine c9q8s0r5e8m4v4l-np-lscpsw-xdz87-qxpvk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-xrcj42xlnjvs-candidate-lscpsw--managed/providers/Microsoft.Compute/virtualMachines/c9q8s0r5e8m4v4l-np-lscpsw-xdz87-qxpvk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lscpsw\" for cluster \"cluster-candidate-5-1-lscpsw\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a793a7485a0\u003e: \n    failed to create NodePool np-lscpsw: failed waiting for nodepool=\"np-lscpsw\" for cluster \"cluster-candidate-5-1-lscpsw\" in resourcegroup=\"rg-candidate-5-1-xrcj42xlnjvs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5813dcb9-1643-4e9b-822d-a0503544e72d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5813dcb9-1643-4e9b-822d-a0503544e72d\",\n      \"name\": \"5813dcb9-1643-4e9b-822d-a0503544e72d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:46:26.081906797Z\",\n      \"endTime\": \"2026-10-07T18:15:34.783886022Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c9q8s0r5e8m4v4l-np-lscpsw-xdz87-b86dg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-xrcj42xlnjvs-candidate-lscpsw--managed/providers/Microsoft.Compute/virtualMachines/c9q8s0r5e8m4v4l-np-lscpsw-xdz87-b86dg to exist\\nMachine c9q8s0r5e8m4v4l-np-lscpsw-xdz87-qxpvk: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-xrcj42xlnjvs-candidate-lscpsw--managed/providers/Microsoft.Compute/virtualMachines/c9q8s0r5e8m4v4l-np-lscpsw-xdz87-qxpvk to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c9q8s0r5e8m4v4l-np-lscpsw-xdz87-b86dg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-xrcj42xlnjvs-candidate-lscpsw--managed/providers/Microsoft.Compute/virtualMachines/c9q8s0r5e8m4v4l-np-lscpsw-xdz87-b86dg to exist\\nMachine c9q8s0r5e8m4v4l-np-lscpsw-xdz87-qxpvk: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-xrcj42xlnjvs-candidate-lscpsw--managed/providers/Microsoft.Compute/virtualMachines/c9q8s0r5e8m4v4l-np-lscpsw-xdz87-qxpvk to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"45d03713b24d3172d508260d87a84cfaf5ceb2ae0c7208f1108fcfd73ac7f196","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"92a56bbeee063c237c54c6a4c494c87e10a3cc5bf6ede5a63dcdfaf9e7e689fc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2kqzsv\" for cluster \"cluster-candidate-4-20-2kqzsv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1103da4b8e00\u003e: \n    failed to create NodePool np-2kqzsv: failed waiting for nodepool=\"np-2kqzsv\" for cluster \"cluster-candidate-4-20-2kqzsv\" in resourcegroup=\"rg-candidate-4-20-hcllc94k97c8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d9788fe4-7e50-4854-8d7b-82e7c83085c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d9788fe4-7e50-4854-8d7b-82e7c83085c6\",\n      \"name\": \"d9788fe4-7e50-4854-8d7b-82e7c83085c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:40:34.472182873Z\",\n      \"endTime\": \"2026-10-07T18:09:34.723474878Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-cr5n4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hcllc94k97c8-candidate-2kqzsv--managed/providers/Microsoft.Compute/virtualMachines/n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-cr5n4 to exist\\nMachine n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-kfzjd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hcllc94k97c8-candidate-2kqzsv--managed/providers/Microsoft.Compute/virtualMachines/n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-kfzjd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-cr5n4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hcllc94k97c8-candidate-2kqzsv--managed/providers/Microsoft.Compute/virtualMachines/n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-cr5n4 to exist\\nMachine n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-kfzjd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hcllc94k97c8-candidate-2kqzsv--managed/providers/Microsoft.Compute/virtualMachines/n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-kfzjd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2kqzsv\" for cluster \"cluster-candidate-4-20-2kqzsv\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1103da4b8e00\u003e: \n    failed to create NodePool np-2kqzsv: failed waiting for nodepool=\"np-2kqzsv\" for cluster \"cluster-candidate-4-20-2kqzsv\" in resourcegroup=\"rg-candidate-4-20-hcllc94k97c8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d9788fe4-7e50-4854-8d7b-82e7c83085c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d9788fe4-7e50-4854-8d7b-82e7c83085c6\",\n      \"name\": \"d9788fe4-7e50-4854-8d7b-82e7c83085c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:40:34.472182873Z\",\n      \"endTime\": \"2026-10-07T18:09:34.723474878Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-cr5n4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hcllc94k97c8-candidate-2kqzsv--managed/providers/Microsoft.Compute/virtualMachines/n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-cr5n4 to exist\\nMachine n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-kfzjd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hcllc94k97c8-candidate-2kqzsv--managed/providers/Microsoft.Compute/virtualMachines/n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-kfzjd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-cr5n4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hcllc94k97c8-candidate-2kqzsv--managed/providers/Microsoft.Compute/virtualMachines/n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-cr5n4 to exist\\nMachine n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-kfzjd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-hcllc94k97c8-candidate-2kqzsv--managed/providers/Microsoft.Compute/virtualMachines/n7o5u6t2t9p0s8z-np-2kqzsv-9clt4-kfzjd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"673d5f7916baa77f1372c7b0c4c5760239c3edecb389275938cfd1d24f73514f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"f60ae0ea29618f99864000653c981584b53aa5f616efd1c3c85d5021999710ac","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nb62wj\" for cluster \"cluster-candidate-4-23-nb62wj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d66910451c0\u003e: \n    failed to create NodePool np-nb62wj: failed waiting for nodepool=\"np-nb62wj\" for cluster \"cluster-candidate-4-23-nb62wj\" in resourcegroup=\"rg-candidate-4-23-grg7vn74klw9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d174db36-7fc5-4f12-a2d6-87f012680f99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d174db36-7fc5-4f12-a2d6-87f012680f99\",\n      \"name\": \"d174db36-7fc5-4f12-a2d6-87f012680f99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:44:23.305815076Z\",\n      \"endTime\": \"2026-10-07T18:13:24.731351337Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-7m58r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-grg7vn74klw9-candidate-nb62wj--managed/providers/Microsoft.Compute/virtualMachines/w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-7m58r to exist\\nMachine w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-8mkbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-grg7vn74klw9-candidate-nb62wj--managed/providers/Microsoft.Compute/virtualMachines/w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-8mkbb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-7m58r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-grg7vn74klw9-candidate-nb62wj--managed/providers/Microsoft.Compute/virtualMachines/w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-7m58r to exist\\nMachine w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-8mkbb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-grg7vn74klw9-candidate-nb62wj--managed/providers/Microsoft.Compute/virtualMachines/w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-8mkbb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nb62wj\" for cluster \"cluster-candidate-4-23-nb62wj\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d66910451c0\u003e: \n    failed to create NodePool np-nb62wj: failed waiting for nodepool=\"np-nb62wj\" for cluster \"cluster-candidate-4-23-nb62wj\" in resourcegroup=\"rg-candidate-4-23-grg7vn74klw9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d174db36-7fc5-4f12-a2d6-87f012680f99\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/d174db36-7fc5-4f12-a2d6-87f012680f99\",\n      \"name\": \"d174db36-7fc5-4f12-a2d6-87f012680f99\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:44:23.305815076Z\",\n      \"endTime\": \"2026-10-07T18:13:24.731351337Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-7m58r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-grg7vn74klw9-candidate-nb62wj--managed/providers/Microsoft.Compute/virtualMachines/w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-7m58r to exist\\nMachine w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-8mkbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-grg7vn74klw9-candidate-nb62wj--managed/providers/Microsoft.Compute/virtualMachines/w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-8mkbb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-7m58r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-grg7vn74klw9-candidate-nb62wj--managed/providers/Microsoft.Compute/virtualMachines/w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-7m58r to exist\\nMachine w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-8mkbb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-grg7vn74klw9-candidate-nb62wj--managed/providers/Microsoft.Compute/virtualMachines/w6t0v4x4s5n3m8e-np-nb62wj-wp9jt-8mkbb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8a7f21b8efbe558991e039b1d21a53c07bb9f2067baa99cec92d534f581a0771","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"02820abf0337fcd30dd807581ed2e7cced4ca7f03dbbf80f30a8a902b7db9b1a","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x13d15dfbe0a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-pzwq2wsxtbjq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/eff75568-ebfe-4e11-bf54-098440849979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/eff75568-ebfe-4e11-bf54-098440849979\",\n      \"name\": \"eff75568-ebfe-4e11-bf54-098440849979\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:15:36.806023423Z\",\n      \"endTime\": \"2026-10-07T17:44:44.250099219Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-m6k6m-h4k42: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-pzwq2wsxtbjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m6k6m-h4k42 to exist\\nMachine agg-cluster-agg-np-a-m6k6m-wqcdr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-m6k6m-wqcdr: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-m6k6m-h4k42: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-pzwq2wsxtbjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m6k6m-h4k42 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x13d15dfbe0a8\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-pzwq2wsxtbjq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/eff75568-ebfe-4e11-bf54-098440849979\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/eff75568-ebfe-4e11-bf54-098440849979\",\n      \"name\": \"eff75568-ebfe-4e11-bf54-098440849979\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:15:36.806023423Z\",\n      \"endTime\": \"2026-10-07T17:44:44.250099219Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-m6k6m-h4k42: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-pzwq2wsxtbjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m6k6m-h4k42 to exist\\nMachine agg-cluster-agg-np-a-m6k6m-wqcdr: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-m6k6m-wqcdr: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-m6k6m-h4k42: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-pzwq2wsxtbjq--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-m6k6m-h4k42 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"a1a1753db6ef43c508c2ac8b0ab17d1339c20b36f0118463fea8ce7b882748de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"987d0ce54e6e4418fc185183edfbc093147759ab1307a6f8a59df037a4923be7","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x206ba567a560\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-5jrlb2b5r7kp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b896af0-0d09-4a15-8405-35074545d1b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b896af0-0d09-4a15-8405-35074545d1b6\",\n      \"name\": \"1b896af0-0d09-4a15-8405-35074545d1b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:13:20.764000805Z\",\n      \"endTime\": \"2026-10-07T17:42:24.218659918Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j4c7y2m5q3o0j6f-np-1-szwxx-f74v6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5jrlb2b5r7kp--managed/providers/Microsoft.Compute/virtualMachines/j4c7y2m5q3o0j6f-np-1-szwxx-f74v6 to exist\\nMachine j4c7y2m5q3o0j6f-np-1-szwxx-tbspd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5jrlb2b5r7kp--managed/providers/Microsoft.Compute/virtualMachines/j4c7y2m5q3o0j6f-np-1-szwxx-tbspd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j4c7y2m5q3o0j6f-np-1-szwxx-f74v6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5jrlb2b5r7kp--managed/providers/Microsoft.Compute/virtualMachines/j4c7y2m5q3o0j6f-np-1-szwxx-f74v6 to exist\\nMachine j4c7y2m5q3o0j6f-np-1-szwxx-tbspd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5jrlb2b5r7kp--managed/providers/Microsoft.Compute/virtualMachines/j4c7y2m5q3o0j6f-np-1-szwxx-tbspd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x206ba567a560\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-5jrlb2b5r7kp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b896af0-0d09-4a15-8405-35074545d1b6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/1b896af0-0d09-4a15-8405-35074545d1b6\",\n      \"name\": \"1b896af0-0d09-4a15-8405-35074545d1b6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:13:20.764000805Z\",\n      \"endTime\": \"2026-10-07T17:42:24.218659918Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine j4c7y2m5q3o0j6f-np-1-szwxx-f74v6: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5jrlb2b5r7kp--managed/providers/Microsoft.Compute/virtualMachines/j4c7y2m5q3o0j6f-np-1-szwxx-f74v6 to exist\\nMachine j4c7y2m5q3o0j6f-np-1-szwxx-tbspd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5jrlb2b5r7kp--managed/providers/Microsoft.Compute/virtualMachines/j4c7y2m5q3o0j6f-np-1-szwxx-tbspd to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine j4c7y2m5q3o0j6f-np-1-szwxx-f74v6: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5jrlb2b5r7kp--managed/providers/Microsoft.Compute/virtualMachines/j4c7y2m5q3o0j6f-np-1-szwxx-f74v6 to exist\\nMachine j4c7y2m5q3o0j6f-np-1-szwxx-tbspd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-5jrlb2b5r7kp--managed/providers/Microsoft.Compute/virtualMachines/j4c7y2m5q3o0j6f-np-1-szwxx-tbspd to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b30041f6e3db1fc91ba8eb86f95d4c81eabf28b03c2cc0c190230addd367d619","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"d79148cbc396e3fb2417bb6a5c08546a4bc7626d2cb1d2b265e599d0466ee183","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lcwd68\" for cluster \"cluster-candidate-4-21-lcwd68\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c328d6f3460\u003e: \n    failed to create NodePool np-lcwd68: failed waiting for nodepool=\"np-lcwd68\" for cluster \"cluster-candidate-4-21-lcwd68\" in resourcegroup=\"rg-candidate-4-21-rrhmm78jhkht\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afb758ed-7109-4c7f-9f48-4c3c8f3bcd88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afb758ed-7109-4c7f-9f48-4c3c8f3bcd88\",\n      \"name\": \"afb758ed-7109-4c7f-9f48-4c3c8f3bcd88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:43:13.338439783Z\",\n      \"endTime\": \"2026-10-07T18:12:14.670254845Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5c7e1e4t0q5v9a-np-lcwd68-4bprc-gxzgv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rrhmm78jhkht-candidate-lcwd68--managed/providers/Microsoft.Compute/virtualMachines/b5c7e1e4t0q5v9a-np-lcwd68-4bprc-gxzgv to exist\\nMachine b5c7e1e4t0q5v9a-np-lcwd68-4bprc-zwhlg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rrhmm78jhkht-candidate-lcwd68--managed/providers/Microsoft.Compute/virtualMachines/b5c7e1e4t0q5v9a-np-lcwd68-4bprc-zwhlg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5c7e1e4t0q5v9a-np-lcwd68-4bprc-gxzgv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rrhmm78jhkht-candidate-lcwd68--managed/providers/Microsoft.Compute/virtualMachines/b5c7e1e4t0q5v9a-np-lcwd68-4bprc-gxzgv to exist\\nMachine b5c7e1e4t0q5v9a-np-lcwd68-4bprc-zwhlg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rrhmm78jhkht-candidate-lcwd68--managed/providers/Microsoft.Compute/virtualMachines/b5c7e1e4t0q5v9a-np-lcwd68-4bprc-zwhlg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-lcwd68\" for cluster \"cluster-candidate-4-21-lcwd68\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c328d6f3460\u003e: \n    failed to create NodePool np-lcwd68: failed waiting for nodepool=\"np-lcwd68\" for cluster \"cluster-candidate-4-21-lcwd68\" in resourcegroup=\"rg-candidate-4-21-rrhmm78jhkht\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afb758ed-7109-4c7f-9f48-4c3c8f3bcd88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/afb758ed-7109-4c7f-9f48-4c3c8f3bcd88\",\n      \"name\": \"afb758ed-7109-4c7f-9f48-4c3c8f3bcd88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:43:13.338439783Z\",\n      \"endTime\": \"2026-10-07T18:12:14.670254845Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b5c7e1e4t0q5v9a-np-lcwd68-4bprc-gxzgv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rrhmm78jhkht-candidate-lcwd68--managed/providers/Microsoft.Compute/virtualMachines/b5c7e1e4t0q5v9a-np-lcwd68-4bprc-gxzgv to exist\\nMachine b5c7e1e4t0q5v9a-np-lcwd68-4bprc-zwhlg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rrhmm78jhkht-candidate-lcwd68--managed/providers/Microsoft.Compute/virtualMachines/b5c7e1e4t0q5v9a-np-lcwd68-4bprc-zwhlg to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b5c7e1e4t0q5v9a-np-lcwd68-4bprc-gxzgv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rrhmm78jhkht-candidate-lcwd68--managed/providers/Microsoft.Compute/virtualMachines/b5c7e1e4t0q5v9a-np-lcwd68-4bprc-gxzgv to exist\\nMachine b5c7e1e4t0q5v9a-np-lcwd68-4bprc-zwhlg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-rrhmm78jhkht-candidate-lcwd68--managed/providers/Microsoft.Compute/virtualMachines/b5c7e1e4t0q5v9a-np-lcwd68-4bprc-zwhlg to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b3d4a0ddbbe5003607d0432e2e51c97dc15bc3b3a5acce5b7e6056043a3fc8c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"5cdf839fb7474b7c4643969bf85e742d6e7999ab72dceb75b236195841c8a7ec","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-p7gdrb\" for cluster \"cluster-candidate-5-0-p7gdrb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2fb1d5f71ba0\u003e: \n    failed to create NodePool np-p7gdrb: failed waiting for nodepool=\"np-p7gdrb\" for cluster \"cluster-candidate-5-0-p7gdrb\" in resourcegroup=\"rg-candidate-5-0-pc4wqgt64lq4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4c46141b-f29e-4146-896e-882152b7eceb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4c46141b-f29e-4146-896e-882152b7eceb\",\n      \"name\": \"4c46141b-f29e-4146-896e-882152b7eceb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:45:14.537982049Z\",\n      \"endTime\": \"2026-10-07T18:14:14.681314604Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-59vxb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pc4wqgt64lq4-candidate-p7gdrb--managed/providers/Microsoft.Compute/virtualMachines/p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-59vxb to exist\\nMachine p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-g9q8s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pc4wqgt64lq4-candidate-p7gdrb--managed/providers/Microsoft.Compute/virtualMachines/p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-g9q8s to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-59vxb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pc4wqgt64lq4-candidate-p7gdrb--managed/providers/Microsoft.Compute/virtualMachines/p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-59vxb to exist\\nMachine p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-g9q8s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pc4wqgt64lq4-candidate-p7gdrb--managed/providers/Microsoft.Compute/virtualMachines/p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-g9q8s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-p7gdrb\" for cluster \"cluster-candidate-5-0-p7gdrb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2fb1d5f71ba0\u003e: \n    failed to create NodePool np-p7gdrb: failed waiting for nodepool=\"np-p7gdrb\" for cluster \"cluster-candidate-5-0-p7gdrb\" in resourcegroup=\"rg-candidate-5-0-pc4wqgt64lq4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4c46141b-f29e-4146-896e-882152b7eceb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/4c46141b-f29e-4146-896e-882152b7eceb\",\n      \"name\": \"4c46141b-f29e-4146-896e-882152b7eceb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:45:14.537982049Z\",\n      \"endTime\": \"2026-10-07T18:14:14.681314604Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-59vxb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pc4wqgt64lq4-candidate-p7gdrb--managed/providers/Microsoft.Compute/virtualMachines/p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-59vxb to exist\\nMachine p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-g9q8s: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pc4wqgt64lq4-candidate-p7gdrb--managed/providers/Microsoft.Compute/virtualMachines/p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-g9q8s to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-59vxb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pc4wqgt64lq4-candidate-p7gdrb--managed/providers/Microsoft.Compute/virtualMachines/p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-59vxb to exist\\nMachine p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-g9q8s: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-pc4wqgt64lq4-candidate-p7gdrb--managed/providers/Microsoft.Compute/virtualMachines/p2s0l6s4q6p5x7g-np-p7gdrb-pqx5j-g9q8s to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b6af569cea0c44324073851a7be2cfe2de59e5748936d8037b83bc368e936cf6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"9a80126b88b88f73aaed428e993ee3eb35ad2dc5ae4652821263df2cf667dc44","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x195b047b05e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-hkmgcjdzqcqq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/bdff2772-3fa6-4abc-9d11-2c48138ff9bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/bdff2772-3fa6-4abc-9d11-2c48138ff9bc\",\n      \"name\": \"bdff2772-3fa6-4abc-9d11-2c48138ff9bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:14:56.224189159Z\",\n      \"endTime\": \"2026-10-07T17:44:04.263637459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-jttqc-rd98n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-hkmgcjdzqcqq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-jttqc-rd98n to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-jttqc-rd98n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-hkmgcjdzqcqq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-jttqc-rd98n to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x195b047b05e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-hkmgcjdzqcqq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/bdff2772-3fa6-4abc-9d11-2c48138ff9bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/bdff2772-3fa6-4abc-9d11-2c48138ff9bc\",\n      \"name\": \"bdff2772-3fa6-4abc-9d11-2c48138ff9bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:14:56.224189159Z\",\n      \"endTime\": \"2026-10-07T17:44:04.263637459Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-jttqc-rd98n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-hkmgcjdzqcqq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-jttqc-rd98n to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-jttqc-rd98n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-hkmgcjdzqcqq--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-jttqc-rd98n to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"de85aeaf4d8e9d758ae318b4e3ee84225bb94dc38404510b2cec936b3ab191be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"522a7104067322270c1dd86403e8358453c5864b0e27aac4cc5b395bb3bb08b1","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x21e4baf3c498\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gpqfttn5vrxt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0bfed0f8-a294-4f34-9bc1-112faf5a4865\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0bfed0f8-a294-4f34-9bc1-112faf5a4865\",\n      \"name\": \"0bfed0f8-a294-4f34-9bc1-112faf5a4865\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:15:04.407912197Z\",\n      \"endTime\": \"2026-10-07T17:44:04.236306088Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-wcplc-b8dk7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-gpqfttn5vrxt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-wcplc-b8dk7 to exist\\nMachine cilium-cluster-cilium-np-wcplc-spjl7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-gpqfttn5vrxt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-wcplc-spjl7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-wcplc-b8dk7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-gpqfttn5vrxt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-wcplc-b8dk7 to exist\\nMachine cilium-cluster-cilium-np-wcplc-spjl7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-gpqfttn5vrxt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-wcplc-spjl7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x21e4baf3c498\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-gpqfttn5vrxt\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0bfed0f8-a294-4f34-9bc1-112faf5a4865\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/0bfed0f8-a294-4f34-9bc1-112faf5a4865\",\n      \"name\": \"0bfed0f8-a294-4f34-9bc1-112faf5a4865\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:15:04.407912197Z\",\n      \"endTime\": \"2026-10-07T17:44:04.236306088Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-wcplc-b8dk7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-gpqfttn5vrxt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-wcplc-b8dk7 to exist\\nMachine cilium-cluster-cilium-np-wcplc-spjl7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-gpqfttn5vrxt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-wcplc-spjl7 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-wcplc-b8dk7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-gpqfttn5vrxt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-wcplc-b8dk7 to exist\\nMachine cilium-cluster-cilium-np-wcplc-spjl7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-gpqfttn5vrxt--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-wcplc-spjl7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e31b29bdb827d15b547d35ebbac84398aa5d0623a0d92bddce6e8fbc388c6d27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107872150416265216","occurred_at":"2026-10-07T16:34:17Z","failed_at":"e2e","signature_id":"e8b902ffd47c497bcda3d3146c3d7891d7784d3563fc15636c6894b1f96e87aa","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cq8x8c\" for cluster \"cluster-candidate-4-22-cq8x8c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b2b51738800\u003e: \n    failed to create NodePool np-cq8x8c: failed waiting for nodepool=\"np-cq8x8c\" for cluster \"cluster-candidate-4-22-cq8x8c\" in resourcegroup=\"rg-candidate-4-22-jv6xxmcszfmz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5dc58718-cb83-4ae7-9c5f-3112077aac0f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5dc58718-cb83-4ae7-9c5f-3112077aac0f\",\n      \"name\": \"5dc58718-cb83-4ae7-9c5f-3112077aac0f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:43:42.061217381Z\",\n      \"endTime\": \"2026-10-07T18:12:44.656595031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-cz892: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jv6xxmcszfmz-candidate-cq8x8c--managed/providers/Microsoft.Compute/virtualMachines/q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-cz892 to exist\\nMachine q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-tscrv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jv6xxmcszfmz-candidate-cq8x8c--managed/providers/Microsoft.Compute/virtualMachines/q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-tscrv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-cz892: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jv6xxmcszfmz-candidate-cq8x8c--managed/providers/Microsoft.Compute/virtualMachines/q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-cz892 to exist\\nMachine q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-tscrv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jv6xxmcszfmz-candidate-cq8x8c--managed/providers/Microsoft.Compute/virtualMachines/q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-tscrv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-cq8x8c\" for cluster \"cluster-candidate-4-22-cq8x8c\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b2b51738800\u003e: \n    failed to create NodePool np-cq8x8c: failed waiting for nodepool=\"np-cq8x8c\" for cluster \"cluster-candidate-4-22-cq8x8c\" in resourcegroup=\"rg-candidate-4-22-jv6xxmcszfmz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5dc58718-cb83-4ae7-9c5f-3112077aac0f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/brazilsouth/hcpOperationStatuses/5dc58718-cb83-4ae7-9c5f-3112077aac0f\",\n      \"name\": \"5dc58718-cb83-4ae7-9c5f-3112077aac0f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T17:43:42.061217381Z\",\n      \"endTime\": \"2026-10-07T18:12:44.656595031Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-cz892: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jv6xxmcszfmz-candidate-cq8x8c--managed/providers/Microsoft.Compute/virtualMachines/q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-cz892 to exist\\nMachine q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-tscrv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jv6xxmcszfmz-candidate-cq8x8c--managed/providers/Microsoft.Compute/virtualMachines/q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-tscrv to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-cz892: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jv6xxmcszfmz-candidate-cq8x8c--managed/providers/Microsoft.Compute/virtualMachines/q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-cz892 to exist\\nMachine q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-tscrv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-jv6xxmcszfmz-candidate-cq8x8c--managed/providers/Microsoft.Compute/virtualMachines/q1a5u8i6s7q3g7y-np-cq8x8c-hpdc9-tscrv to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":11,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743"],"matched_occurrences":11,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107765204698796032","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-07T09:29:19Z","started_at":"2026-10-07T09:29:19Z","completed_at":"2026-10-07T12:43:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-07T12:54:45.548901546Z","timing_metadata_checked_at":"2026-10-07T12:54:45.548901546Z","region":"eastus2euap","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-07T12:54:45.551672337Z","region_metadata_checked_at":"2026-10-07T12:54:45.551672337Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9e361e5ef2be685ce094391cc79639c81ee9003de1eea53a4cf168cfb6e938f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107765204698796032","occurred_at":"2026-10-07T09:29:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-07T09:23:41Z","started_at":"2026-10-07T09:23:41Z","completed_at":"2026-10-07T12:43:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-07T12:54:45.526047215Z","timing_metadata_checked_at":"2026-10-07T12:54:45.526047215Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-07T12:54:45.529687504Z","region_metadata_checked_at":"2026-10-07T13:59:18.152286631Z"},"failed_at":["e2e"],"failed_test_count":60,"occurrences":[{"row_id":"003738ae87ab193e0668d42448f7f2aed8cb99f7500b093c883849b984529242","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"ed9355468df05ca46faf44afb5f61337814d9c2ece35b6740bc25165e7398be6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6g2vplsk2mk7/cluster-candidate-5-0-h6qmxk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x927ddf70460\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-h6qmxk: failed waiting for cluster=\"cluster-candidate-5-0-h6qmxk\" in resourcegroup=\"rg-candidate-5-0-6g2vplsk2mk7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9f5559ea-24ce-464c-ba42-f4702aefacd5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9f5559ea-24ce-464c-ba42-f4702aefacd5\",\n      \"name\": \"9f5559ea-24ce-464c-ba42-f4702aefacd5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:14.87447448Z\",\n      \"endTime\": \"2026-10-07T11:13:14.934598428Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-6g2vplsk2mk7/cluster-candidate-5-0-h6qmxk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x927ddf70460\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-h6qmxk: failed waiting for cluster=\"cluster-candidate-5-0-h6qmxk\" in resourcegroup=\"rg-candidate-5-0-6g2vplsk2mk7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9f5559ea-24ce-464c-ba42-f4702aefacd5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9f5559ea-24ce-464c-ba42-f4702aefacd5\",\n      \"name\": \"9f5559ea-24ce-464c-ba42-f4702aefacd5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:14.87447448Z\",\n      \"endTime\": \"2026-10-07T11:13:14.934598428Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0c4999d7dd1cb9dfe1090bca37a73f64ecf5b642404a17628deec85b2b177575","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"85089d90025f7e4b136612cbe3126cdeb157f81d89a6d1f85bf84e1242f59f52","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x341fe2b86760\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ptq5zsv79s42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/53b78cd8-ac9a-4444-a5d8-4c19b93bf968\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/53b78cd8-ac9a-4444-a5d8-4c19b93bf968\",\n      \"name\": \"53b78cd8-ac9a-4444-a5d8-4c19b93bf968\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:55.810865072Z\",\n      \"endTime\": \"2026-10-07T10:18:04.487764754Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0x341fe2b86760\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ptq5zsv79s42\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/53b78cd8-ac9a-4444-a5d8-4c19b93bf968\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/53b78cd8-ac9a-4444-a5d8-4c19b93bf968\",\n      \"name\": \"53b78cd8-ac9a-4444-a5d8-4c19b93bf968\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:55.810865072Z\",\n      \"endTime\": \"2026-10-07T10:18:04.487764754Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"10f815ade2c35c5a08a38d415c6d374dde9b18f4c0ad1c1602479e3457d56c3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"2a9264902cac7135ae10487055f2ec90d442d6082711c47d54f8a83eaf00eefd","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x186c1f59fe00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-xkrgtz49hrh4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xkrgtz49hrh4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0x186c1f59fe00\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-xkrgtz49hrh4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-xkrgtz49hrh4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"171712904d2de59e4384b399f009a85e0782c312fb6a05f257bcab3e8eb4a7e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"9f87a1d4862080e4d4b05c636d70abf9d6119ecf1dc420ad540a69449802aba5","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x20bebe634820\u003e: \n    failed to create HCP cluster ea-listrlzp2p: failed waiting for cluster=\"ea-listrlzp2p\" in resourcegroup=\"ea-list-k4w9b29d9847\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07411f45-babd-4a30-b6ee-3bba10d43fa1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07411f45-babd-4a30-b6ee-3bba10d43fa1\",\n      \"name\": \"07411f45-babd-4a30-b6ee-3bba10d43fa1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:47:32.468226037Z\",\n      \"endTime\": \"2026-10-07T11:16:34.835170504Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x20bebe634820\u003e: \n    failed to create HCP cluster ea-listrlzp2p: failed waiting for cluster=\"ea-listrlzp2p\" in resourcegroup=\"ea-list-k4w9b29d9847\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07411f45-babd-4a30-b6ee-3bba10d43fa1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/07411f45-babd-4a30-b6ee-3bba10d43fa1\",\n      \"name\": \"07411f45-babd-4a30-b6ee-3bba10d43fa1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:47:32.468226037Z\",\n      \"endTime\": \"2026-10-07T11:16:34.835170504Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"19aed2c09f33f2451badfeb125475c1117dcda75ca01c9fca9413b19b521b3f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"a1712b1999aa403be90c2c39032dbf3482175f4135dafb21660a6280946668f1","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15d5627c8480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-jq7pzjw2cp54\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6fe1a352-ffa7-4b41-9c47-d5903779eadf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6fe1a352-ffa7-4b41-9c47-d5903779eadf\",\n      \"name\": \"6fe1a352-ffa7-4b41-9c47-d5903779eadf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:08.895469535Z\",\n      \"endTime\": \"2026-10-07T10:21:14.61315509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:84]: failed to create HCP cluster for patch-name test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x15d5627c8480\u003e: \n    failed to create HCP cluster patch-name-cluster: failed waiting for cluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-jq7pzjw2cp54\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6fe1a352-ffa7-4b41-9c47-d5903779eadf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6fe1a352-ffa7-4b41-9c47-d5903779eadf\",\n      \"name\": \"6fe1a352-ffa7-4b41-9c47-d5903779eadf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:08.895469535Z\",\n      \"endTime\": \"2026-10-07T10:21:14.61315509Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1edcfb5e56759bc0d8da5ae2332b371bf9175a2ce16d63eddb7970c91d37ea1d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"38424e0b4e68892bec2eb668127066c6f2a26c76ecd81204afaf71f81b811e31","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xde11fb947e0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0c5a96-73a7-46a1-ae56-141bff5a9cb0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0c5a96-73a7-46a1-ae56-141bff5a9cb0\",\n      \"name\": \"3c0c5a96-73a7-46a1-ae56-141bff5a9cb0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:49:41.529698029Z\",\n      \"endTime\": \"2026-10-07T10:18:44.600218991Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n    \u003c*exported.ResponseError | 0xde11fb947e0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0c5a96-73a7-46a1-ae56-141bff5a9cb0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3c0c5a96-73a7-46a1-ae56-141bff5a9cb0\",\n      \"name\": \"3c0c5a96-73a7-46a1-ae56-141bff5a9cb0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:49:41.529698029Z\",\n      \"endTime\": \"2026-10-07T10:18:44.600218991Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2309f2c077f1d07089afa41a6eef1b12d48a0da74978291c2a99f8111c65e24b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"6bb41cca9dbaab1d3f7795fa9b64ec70faea8e4b1d05cd73381f7f45ac480051","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3437203728c0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mgbhjw52mcvc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e9424b8-7f1c-40c2-82b4-31517d779d0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e9424b8-7f1c-40c2-82b4-31517d779d0b\",\n      \"name\": \"6e9424b8-7f1c-40c2-82b4-31517d779d0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:54.433534766Z\",\n      \"endTime\": \"2026-10-07T10:17:54.588329133Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3437203728c0\u003e: \n    failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-mgbhjw52mcvc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e9424b8-7f1c-40c2-82b4-31517d779d0b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6e9424b8-7f1c-40c2-82b4-31517d779d0b\",\n      \"name\": \"6e9424b8-7f1c-40c2-82b4-31517d779d0b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:54.433534766Z\",\n      \"endTime\": \"2026-10-07T10:17:54.588329133Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"23864a9aaff48a9a64176fa295403d06e7644c446224d1e27a1955a07952d97a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"ea3cca5efa4a4d974d95cceb3e9779fac77cf63503170b6ca9e52bfc03e01dac","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14682ff371e0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5tccgln57r4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5cae4d34-c9f9-4fce-84cc-e6e26de3b174\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5cae4d34-c9f9-4fce-84cc-e6e26de3b174\",\n      \"name\": \"5cae4d34-c9f9-4fce-84cc-e6e26de3b174\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:14.268574552Z\",\n      \"endTime\": \"2026-10-07T12:08:14.728480249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x14682ff371e0\u003e: \n    failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5tccgln57r4g\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5cae4d34-c9f9-4fce-84cc-e6e26de3b174\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5cae4d34-c9f9-4fce-84cc-e6e26de3b174\",\n      \"name\": \"5cae4d34-c9f9-4fce-84cc-e6e26de3b174\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:14.268574552Z\",\n      \"endTime\": \"2026-10-07T12:08:14.728480249Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"23f6c4ff164445f9de9ae934ed114b0ac013514ef3f1ac7947c66056e42bffef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"8b0eef9c1f0b24176611911df5aa0d480ab4a2120a5657f9316084eca9ea0bdf","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e0fc50a4480\u003e: \n    failed to create HCP cluster np-downgrade-s292rk: failed waiting for cluster=\"np-downgrade-s292rk\" in resourcegroup=\"rg-np-downgrade-s292rk-zk4jmh695nzj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c877c7b1-118a-45af-95c5-bf8e831b2f1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c877c7b1-118a-45af-95c5-bf8e831b2f1d\",\n      \"name\": \"c877c7b1-118a-45af-95c5-bf8e831b2f1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:40:59.843518116Z\",\n      \"endTime\": \"2026-10-07T12:10:04.833949367Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e0fc50a4480\u003e: \n    failed to create HCP cluster np-downgrade-s292rk: failed waiting for cluster=\"np-downgrade-s292rk\" in resourcegroup=\"rg-np-downgrade-s292rk-zk4jmh695nzj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c877c7b1-118a-45af-95c5-bf8e831b2f1d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c877c7b1-118a-45af-95c5-bf8e831b2f1d\",\n      \"name\": \"c877c7b1-118a-45af-95c5-bf8e831b2f1d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:40:59.843518116Z\",\n      \"endTime\": \"2026-10-07T12:10:04.833949367Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"26525db74f99065351d20b06b9befa05f91317a1601aa9be40d548283722f9c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"14135c0f42e6038fec848d3d13ac98a506c6bb5e44786249455cbf6831ee7ee5","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2851e862a620\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ws854cfjbfc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"name\": \"5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:27.70826866Z\",\n      \"endTime\": \"2026-10-07T12:10:34.84968826Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:272]: failed to create HCP cluster shoebox-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2851e862a620\u003e: \n    failed to create HCP cluster shoebox-hcp-cluster: failed waiting for cluster=\"shoebox-hcp-cluster\" in resourcegroup=\"shoebox-cluster-ws854cfjbfc7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"name\": \"5d35187f-9481-4680-bc88-1fa2a5bd4bb4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:27.70826866Z\",\n      \"endTime\": \"2026-10-07T12:10:34.84968826Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2716381df736ca49506aaa4081f83b63cba0b7570b526ad0cf211df861b5b8e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"4285d36e41c52c52f14f33e255d32a9fbefe892fb4555c01f3990db62c8bb7f7","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5e92b4ffb60\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jsrm64gc6pjs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jsrm64gc6pjs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5e92b4ffb60\u003e: \n    failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jsrm64gc6pjs\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jsrm64gc6pjs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"2b97fbdccf98142defe8f61c89dcf270547ef9febdc09e0d026f65c34547247b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"e68e84c5ef68c6d5ed89d88a519e423d565bc4c0fe60d49c95bf93910fc3b5ac","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37dd151f8520\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-6cgsv65nx9n7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/34b0c2af-8aaa-4f79-9413-e75d8e7d93a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/34b0c2af-8aaa-4f79-9413-e75d8e7d93a2\",\n      \"name\": \"34b0c2af-8aaa-4f79-9413-e75d8e7d93a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:50:14.090475934Z\",\n      \"endTime\": \"2026-10-07T10:19:14.520219506Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37dd151f8520\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-6cgsv65nx9n7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/34b0c2af-8aaa-4f79-9413-e75d8e7d93a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/34b0c2af-8aaa-4f79-9413-e75d8e7d93a2\",\n      \"name\": \"34b0c2af-8aaa-4f79-9413-e75d8e7d93a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:50:14.090475934Z\",\n      \"endTime\": \"2026-10-07T10:19:14.520219506Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2d4ae5f8a5a927d1d1f34e39e944535afd74719753c26c4ea8fc0d5eabaeca42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"f62dfbfab386ba992afc971567b9d0d76b1b0e0237b5e0a68a1b3ec19e0c909f","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb405b54e780\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xwt6pngn9s2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39973bec-a044-461c-a50c-080b9b4ce44e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39973bec-a044-461c-a50c-080b9b4ce44e\",\n      \"name\": \"39973bec-a044-461c-a50c-080b9b4ce44e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:24.877409448Z\",\n      \"endTime\": \"2026-10-07T12:08:24.807664199Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xb405b54e780\u003e: \n    failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-xwt6pngn9s2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39973bec-a044-461c-a50c-080b9b4ce44e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/39973bec-a044-461c-a50c-080b9b4ce44e\",\n      \"name\": \"39973bec-a044-461c-a50c-080b9b4ce44e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:24.877409448Z\",\n      \"endTime\": \"2026-10-07T12:08:24.807664199Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"362b1b08d042b906f719fde8c020c44a3e63309a323f775d1bc857080a2a8b9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"161f29459dd25be2b9948cb9b2928ee7923fcc1e8cff6ecba8619d7726e0f54d","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x231265410c40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-m6pl2gptcdfp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"name\": \"3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:27.398564047Z\",\n      \"endTime\": \"2026-10-07T12:08:34.791319585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0x231265410c40\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-m6pl2gptcdfp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"name\": \"3147c124-12ef-4d26-acf9-96afbf8f7fdc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:27.398564047Z\",\n      \"endTime\": \"2026-10-07T12:08:34.791319585Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"37599ea60bc0ac91ca94911aba416518785a7af4114d4e0e4e0951706ea7766b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"96927fc5b20d35c1c9f5483135274e1e4c184f0687b9c7836cf8e11f8d5b485d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hkdz2nxrsjhh/cluster-candidate-4-21-d4ssjn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x38f8a2b10240\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-d4ssjn: failed waiting for cluster=\"cluster-candidate-4-21-d4ssjn\" in resourcegroup=\"rg-candidate-4-21-hkdz2nxrsjhh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3579b35f-53ea-4026-90cb-38b7f7202820\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3579b35f-53ea-4026-90cb-38b7f7202820\",\n      \"name\": \"3579b35f-53ea-4026-90cb-38b7f7202820\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:02.416842426Z\",\n      \"endTime\": \"2026-10-07T11:13:04.946994538Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hkdz2nxrsjhh/cluster-candidate-4-21-d4ssjn should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x38f8a2b10240\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-d4ssjn: failed waiting for cluster=\"cluster-candidate-4-21-d4ssjn\" in resourcegroup=\"rg-candidate-4-21-hkdz2nxrsjhh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3579b35f-53ea-4026-90cb-38b7f7202820\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3579b35f-53ea-4026-90cb-38b7f7202820\",\n      \"name\": \"3579b35f-53ea-4026-90cb-38b7f7202820\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:02.416842426Z\",\n      \"endTime\": \"2026-10-07T11:13:04.946994538Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3be0281ccd1fa67af4aab333af6ff79ca15c0aa614df9dae8f3720492f689364","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"977fcb1eb57ae1e979b4c1b950f7d9cc8abf8782c740f514024b50110304411e","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x335c22036160\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7nldjt8fwf95\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff0ed7f2-6772-4f0b-98e5-ec67a1f2a5e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff0ed7f2-6772-4f0b-98e5-ec67a1f2a5e4\",\n      \"name\": \"ff0ed7f2-6772-4f0b-98e5-ec67a1f2a5e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:28.998485099Z\",\n      \"endTime\": \"2026-10-07T10:21:34.495305491Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0x335c22036160\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-7nldjt8fwf95\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff0ed7f2-6772-4f0b-98e5-ec67a1f2a5e4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ff0ed7f2-6772-4f0b-98e5-ec67a1f2a5e4\",\n      \"name\": \"ff0ed7f2-6772-4f0b-98e5-ec67a1f2a5e4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:28.998485099Z\",\n      \"endTime\": \"2026-10-07T10:21:34.495305491Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3c92e7be105c9305feecf4014145c86225954b704af09d9ef378449bca041277","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"ecdadfc64076f4841dd66ae1fae57132eb211d25a58c181f0dbea68197273788","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30510e393840\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-6n2m8kjftrjk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f40a774-0a6b-406f-b3e0-c881c5b48c28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f40a774-0a6b-406f-b3e0-c881c5b48c28\",\n      \"name\": \"1f40a774-0a6b-406f-b3e0-c881c5b48c28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:52.605546648Z\",\n      \"endTime\": \"2026-10-07T10:17:54.505358431Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x30510e393840\u003e: \n    failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-6n2m8kjftrjk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f40a774-0a6b-406f-b3e0-c881c5b48c28\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f40a774-0a6b-406f-b3e0-c881c5b48c28\",\n      \"name\": \"1f40a774-0a6b-406f-b3e0-c881c5b48c28\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:52.605546648Z\",\n      \"endTime\": \"2026-10-07T10:17:54.505358431Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3cfa946996a12e1b60c9c75311c1349707125b6049bd9e5eb85cfbf9d9dadf87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"ff16efcd5d70df29c345cf8295a63cf7efb04beaae9b28016e8a9a1d93d36082","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-hjp4bn\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a46bd3ea760\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-hjp4bn: failed waiting for cluster=\"cp-ystream-upgrade-4-22-hjp4bn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-69pb6s9zfqhj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5b982c67-2a0b-4195-a11b-235682400cd2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5b982c67-2a0b-4195-a11b-235682400cd2\",\n      \"name\": \"5b982c67-2a0b-4195-a11b-235682400cd2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:46:31.219755391Z\",\n      \"endTime\": \"2026-10-07T11:15:34.961415579Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-hjp4bn\" at install version 4.21\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2a46bd3ea760\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-22-hjp4bn: failed waiting for cluster=\"cp-ystream-upgrade-4-22-hjp4bn\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-69pb6s9zfqhj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5b982c67-2a0b-4195-a11b-235682400cd2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5b982c67-2a0b-4195-a11b-235682400cd2\",\n      \"name\": \"5b982c67-2a0b-4195-a11b-235682400cd2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:46:31.219755391Z\",\n      \"endTime\": \"2026-10-07T11:15:34.961415579Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"462b8ed6f1395a00f2baa9a44b8c8c1a0963c567883716fbc552d20c720351fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"f671365b45f055e398b3b3ea370acfc9dc7cbc2da9b1a10f010cc014e9b21b86","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-tqxzz2 at version 4.22.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0x331f2f68e6a0\u003e: \n    failed to create HCP cluster np-skip-minor-tqxzz2: failed waiting for cluster=\"np-skip-minor-tqxzz2\" in resourcegroup=\"rg-np-skip-minor-tqxzz2-4phhgkzvl662\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3bf832de-8a79-4bb8-8b93-10f2de10820f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3bf832de-8a79-4bb8-8b93-10f2de10820f\",\n      \"name\": \"3bf832de-8a79-4bb8-8b93-10f2de10820f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:52.09303905Z\",\n      \"endTime\": \"2026-10-07T12:08:54.828044154Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-tqxzz2 at version 4.22.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0x331f2f68e6a0\u003e: \n    failed to create HCP cluster np-skip-minor-tqxzz2: failed waiting for cluster=\"np-skip-minor-tqxzz2\" in resourcegroup=\"rg-np-skip-minor-tqxzz2-4phhgkzvl662\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3bf832de-8a79-4bb8-8b93-10f2de10820f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3bf832de-8a79-4bb8-8b93-10f2de10820f\",\n      \"name\": \"3bf832de-8a79-4bb8-8b93-10f2de10820f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:52.09303905Z\",\n      \"endTime\": \"2026-10-07T12:08:54.828044154Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"53f25b6f735b4246cc313b9975c59e2b9ef47055e14e5b9f40aa98f3f5b98cc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"2a4544a43b4c4906031ddca838eac98d21615c334e36bf9f03833d3a9649ae82","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-tcv2dg\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a4975a7c5e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-tcv2dg: failed waiting for cluster=\"cp-ystream-upgrade-4-21-tcv2dg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-l28vh755d899\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3c3d8ab-c656-42f4-9688-9bfce51a0648\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3c3d8ab-c656-42f4-9688-9bfce51a0648\",\n      \"name\": \"c3c3d8ab-c656-42f4-9688-9bfce51a0648\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:46:26.13173324Z\",\n      \"endTime\": \"2026-10-07T11:15:34.96005862Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-tcv2dg\" at install version 4.20\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3a4975a7c5e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-21-tcv2dg: failed waiting for cluster=\"cp-ystream-upgrade-4-21-tcv2dg\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-l28vh755d899\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3c3d8ab-c656-42f4-9688-9bfce51a0648\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c3c3d8ab-c656-42f4-9688-9bfce51a0648\",\n      \"name\": \"c3c3d8ab-c656-42f4-9688-9bfce51a0648\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:46:26.13173324Z\",\n      \"endTime\": \"2026-10-07T11:15:34.96005862Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"573f42a5f888a77103262a9a32552f28c8034eac8bb28e97516a95c0c4aef648","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"67e3544022c911074f09ce65702282c6041cff3d54247562a71ba2e779489a2a","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21855c06aa20\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-2bhn4hmftg4v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-2bhn4hmftg4v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x21855c06aa20\u003e: \n    failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-2bhn4hmftg4v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-2bhn4hmftg4v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5bcdd8b6621727a9154a4ab9969f95cd4ad71f7690d84401145d7453649c3b1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"a2ce9a68f555401f462f07278b9c8957cd22840d85da390141086d71898ecd04","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-p6ctw2 with version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26e4dc71a4c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-p6ctw2: failed waiting for cluster=\"np-version-upgrade-cluster-p6ctw2\" in resourcegroup=\"rg-np-version-upgrade-p6ctw2-gxs4lhjp4gfz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f6ade4-81d1-451d-b59d-15dcae59714f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f6ade4-81d1-451d-b59d-15dcae59714f\",\n      \"name\": \"13f6ade4-81d1-451d-b59d-15dcae59714f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:22.119909633Z\",\n      \"endTime\": \"2026-10-07T12:08:24.732534106Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-p6ctw2 with version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x26e4dc71a4c0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-p6ctw2: failed waiting for cluster=\"np-version-upgrade-cluster-p6ctw2\" in resourcegroup=\"rg-np-version-upgrade-p6ctw2-gxs4lhjp4gfz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f6ade4-81d1-451d-b59d-15dcae59714f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/13f6ade4-81d1-451d-b59d-15dcae59714f\",\n      \"name\": \"13f6ade4-81d1-451d-b59d-15dcae59714f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:22.119909633Z\",\n      \"endTime\": \"2026-10-07T12:08:24.732534106Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6eeb3c53fafa24a145d3e226a0a5ab3f92c5c001e8142038d8c60591a8d07254","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"03da4f253bd1559e010815404447df7a21bfdb927bb20e898538689c3b1a7fc2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-z9xhpt9dxztr/cluster-candidate-4-22-8frl8d should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc3c4040eec0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-8frl8d: failed waiting for cluster=\"cluster-candidate-4-22-8frl8d\" in resourcegroup=\"rg-candidate-4-22-z9xhpt9dxztr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"name\": \"6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:02.12350471Z\",\n      \"endTime\": \"2026-10-07T11:13:04.942313802Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-z9xhpt9dxztr/cluster-candidate-4-22-8frl8d should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc3c4040eec0\u003e: \n    failed to create HCP cluster cluster-candidate-4-22-8frl8d: failed waiting for cluster=\"cluster-candidate-4-22-8frl8d\" in resourcegroup=\"rg-candidate-4-22-z9xhpt9dxztr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"name\": \"6731dbc8-8cd4-41ae-bf5b-f0e9ab07cfbf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:02.12350471Z\",\n      \"endTime\": \"2026-10-07T11:13:04.942313802Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7950152f146cd98a49d6851421cf985fb316dd7ad24be5c31f12d8e36286ac1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"b6c92859057095532f3f24d018d39c6d2961d7b695e023087e8f6547d3916117","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36474092e220\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tsgkj9sbfmk9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5c675833-7374-4d56-8296-8d479088785e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5c675833-7374-4d56-8296-8d479088785e\",\n      \"name\": \"5c675833-7374-4d56-8296-8d479088785e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:51:43.574661454Z\",\n      \"endTime\": \"2026-10-07T10:20:44.509181086Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:114]: failed to create HCP cluster for pull secret test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x36474092e220\u003e: \n    failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-tsgkj9sbfmk9\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5c675833-7374-4d56-8296-8d479088785e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5c675833-7374-4d56-8296-8d479088785e\",\n      \"name\": \"5c675833-7374-4d56-8296-8d479088785e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:51:43.574661454Z\",\n      \"endTime\": \"2026-10-07T10:20:44.509181086Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7b68de33b9447ed29a22e902aeefc6aac34bfa6af4396b0641f186301c512e8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"ad1adaad6c931397a5cfb6188dbcbf7aec77f60cfab7d62edf2c9ea2f44522e5","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-v64v5n\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3eb6842062a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-v64v5n: failed waiting for cluster=\"cluster-zstream-4-23-v64v5n\" in resourcegroup=\"rg-zstream-upgrade-4-23-lxr77fqqlcc4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcd5b8-e1cc-4dde-aa1d-4fd166ab6ab6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcd5b8-e1cc-4dde-aa1d-4fd166ab6ab6\",\n      \"name\": \"b0fcd5b8-e1cc-4dde-aa1d-4fd166ab6ab6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:37.55941311Z\",\n      \"endTime\": \"2026-10-07T11:13:44.839201363Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-v64v5n\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3eb6842062a0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-v64v5n: failed waiting for cluster=\"cluster-zstream-4-23-v64v5n\" in resourcegroup=\"rg-zstream-upgrade-4-23-lxr77fqqlcc4\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcd5b8-e1cc-4dde-aa1d-4fd166ab6ab6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/b0fcd5b8-e1cc-4dde-aa1d-4fd166ab6ab6\",\n      \"name\": \"b0fcd5b8-e1cc-4dde-aa1d-4fd166ab6ab6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:37.55941311Z\",\n      \"endTime\": \"2026-10-07T11:13:44.839201363Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e1b6191e9faa1f0e88905755e9bf2f076837c1a44bd203ba5387620a08c219f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"e6282f6819cbae90d2080bcc11689d8d54bca0907cf69387e75370c987b1244d","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-tvjfv8 with version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x566e5e2e580\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-tvjfv8: failed waiting for cluster=\"np-version-upgrade-cluster-tvjfv8\" in resourcegroup=\"rg-np-version-upgrade-tvjfv8-hct95xpq2lzg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f9a95eb-a350-40c6-af2a-e703473e0215\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f9a95eb-a350-40c6-af2a-e703473e0215\",\n      \"name\": \"1f9a95eb-a350-40c6-af2a-e703473e0215\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:41.966281744Z\",\n      \"endTime\": \"2026-10-07T12:08:44.804380288Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-tvjfv8 with version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x566e5e2e580\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-tvjfv8: failed waiting for cluster=\"np-version-upgrade-cluster-tvjfv8\" in resourcegroup=\"rg-np-version-upgrade-tvjfv8-hct95xpq2lzg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f9a95eb-a350-40c6-af2a-e703473e0215\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f9a95eb-a350-40c6-af2a-e703473e0215\",\n      \"name\": \"1f9a95eb-a350-40c6-af2a-e703473e0215\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:41.966281744Z\",\n      \"endTime\": \"2026-10-07T12:08:44.804380288Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7e91824f37f3c9497ffd2875bc6daf7259e31a1c1d3c00b0790506106dda6d5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"9612cc0cf025f89e9420879dadf6053baa6863d7bb5e5eaf07f761e2805945f2","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-xfwltf\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a102b44380\u003e: \n    failed to create HCP cluster np-noedge-xfwltf: failed waiting for cluster=\"np-noedge-xfwltf\" in resourcegroup=\"rg-np-noedge-xfwltf-wckscwtz2vn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75fdac03-68fe-4275-bc47-120c356d3ac3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75fdac03-68fe-4275-bc47-120c356d3ac3\",\n      \"name\": \"75fdac03-68fe-4275-bc47-120c356d3ac3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:51.394799188Z\",\n      \"endTime\": \"2026-10-07T12:08:54.815579704Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-xfwltf\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22a102b44380\u003e: \n    failed to create HCP cluster np-noedge-xfwltf: failed waiting for cluster=\"np-noedge-xfwltf\" in resourcegroup=\"rg-np-noedge-xfwltf-wckscwtz2vn6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75fdac03-68fe-4275-bc47-120c356d3ac3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/75fdac03-68fe-4275-bc47-120c356d3ac3\",\n      \"name\": \"75fdac03-68fe-4275-bc47-120c356d3ac3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:51.394799188Z\",\n      \"endTime\": \"2026-10-07T12:08:54.815579704Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8baf6b70efa1f110b1cd34e8e27b18cfc7bec68870a31948904fdb52396708fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"fee1f7575ffa79a5f721c4a55f609c98395d74d67ae1e00d333ecc7ff0426314","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b5f89a004e0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-nkbzqbx5622d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5a305a1c-863d-4c3c-8dfe-d7cc2d61fb87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5a305a1c-863d-4c3c-8dfe-d7cc2d61fb87\",\n      \"name\": \"5a305a1c-863d-4c3c-8dfe-d7cc2d61fb87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:09.506899414Z\",\n      \"endTime\": \"2026-10-07T10:21:14.622216782Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:96]: failed to create HCP cluster for TLS endpoint test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b5f89a004e0\u003e: \n    failed to create HCP cluster tls-endpoint-hcp-cluster: failed waiting for cluster=\"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-nkbzqbx5622d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5a305a1c-863d-4c3c-8dfe-d7cc2d61fb87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5a305a1c-863d-4c3c-8dfe-d7cc2d61fb87\",\n      \"name\": \"5a305a1c-863d-4c3c-8dfe-d7cc2d61fb87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:09.506899414Z\",\n      \"endTime\": \"2026-10-07T10:21:14.622216782Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d5a66776133d09c33fa763b54e8c208916e8935575353427e50c08ccc4477e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"5401c8357fce4c3d57d0b54d9d0c1f712605e1965171f2bc7a7b99c403d39d17","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39ef57d831e0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-d2rzc85bvm5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6397f521-d69c-40cd-8708-e9c85c21063d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6397f521-d69c-40cd-8708-e9c85c21063d\",\n      \"name\": \"6397f521-d69c-40cd-8708-e9c85c21063d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:48:24.801806415Z\",\n      \"endTime\": \"2026-10-07T11:17:24.840938454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x39ef57d831e0\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-d2rzc85bvm5t\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6397f521-d69c-40cd-8708-e9c85c21063d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6397f521-d69c-40cd-8708-e9c85c21063d\",\n      \"name\": \"6397f521-d69c-40cd-8708-e9c85c21063d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:48:24.801806415Z\",\n      \"endTime\": \"2026-10-07T11:17:24.840938454Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8df2fab187c1a45aeeea4737f5ba5bc4f6365c5fde5e16b07cd04b02d74a20b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"a0a734eda8f093e83a4d94cde244e3c5aa94a9b640ad6a9042a19b8c1730494b","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x664b54ecae0\u003e: \n    failed to create HCP cluster np-dg-minor-v6wk6c: failed waiting for cluster=\"np-dg-minor-v6wk6c\" in resourcegroup=\"rg-np-dg-minor-v6wk6c-j7kpgqfkx4ww\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62536149-77a5-4b5d-a914-13bf7d06200d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62536149-77a5-4b5d-a914-13bf7d06200d\",\n      \"name\": \"62536149-77a5-4b5d-a914-13bf7d06200d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:24.096113734Z\",\n      \"endTime\": \"2026-10-07T12:10:24.728804421Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x664b54ecae0\u003e: \n    failed to create HCP cluster np-dg-minor-v6wk6c: failed waiting for cluster=\"np-dg-minor-v6wk6c\" in resourcegroup=\"rg-np-dg-minor-v6wk6c-j7kpgqfkx4ww\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62536149-77a5-4b5d-a914-13bf7d06200d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/62536149-77a5-4b5d-a914-13bf7d06200d\",\n      \"name\": \"62536149-77a5-4b5d-a914-13bf7d06200d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:24.096113734Z\",\n      \"endTime\": \"2026-10-07T12:10:24.728804421Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ed282bd8600be2cf1b352828052ac6c5c9c6a456c12289a01fc6ae5fec985fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"aa67d6d32793cc52a0a7b7125b70972d054b6703e99dbe1985a2e305c66412a0","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2f70e02c4940\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-z6z886s5vlhv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/80ec89c1-e1d7-4d1c-9156-2ef5cd15b42c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/80ec89c1-e1d7-4d1c-9156-2ef5cd15b42c\",\n      \"name\": \"80ec89c1-e1d7-4d1c-9156-2ef5cd15b42c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:38:48.428505117Z\",\n      \"endTime\": \"2026-10-07T12:07:54.803165518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2f70e02c4940\u003e: \n    failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-z6z886s5vlhv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/80ec89c1-e1d7-4d1c-9156-2ef5cd15b42c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/80ec89c1-e1d7-4d1c-9156-2ef5cd15b42c\",\n      \"name\": \"80ec89c1-e1d7-4d1c-9156-2ef5cd15b42c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:38:48.428505117Z\",\n      \"endTime\": \"2026-10-07T12:07:54.803165518Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8f2c0a1e35d81fbf65695d1adfb68c1490e09cf5eabc14abb5928b6e3b66db09","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"d514516b178028fea78395eeb242c18d910c33a636e520b12e6c30ec48120190","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-8s78nq\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b6789bd29a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-8s78nq: failed waiting for cluster=\"cp-ystream-upgrade-4-23-8s78nq\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-dlp6qsnjnzcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e17afad-249e-411b-9b0c-f5c6396456c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e17afad-249e-411b-9b0c-f5c6396456c6\",\n      \"name\": \"5e17afad-249e-411b-9b0c-f5c6396456c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:46:32.119485946Z\",\n      \"endTime\": \"2026-10-07T11:15:34.955895434Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-8s78nq\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b6789bd29a0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-4-23-8s78nq: failed waiting for cluster=\"cp-ystream-upgrade-4-23-8s78nq\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-dlp6qsnjnzcs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e17afad-249e-411b-9b0c-f5c6396456c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5e17afad-249e-411b-9b0c-f5c6396456c6\",\n      \"name\": \"5e17afad-249e-411b-9b0c-f5c6396456c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:46:32.119485946Z\",\n      \"endTime\": \"2026-10-07T11:15:34.955895434Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8f37dd84cb6db4801269809dd59ce236ba4f8fa3880753b5bc7ead7ff60afdf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"65aa41b9d85ed7b6471c3cae1678d3b40aedc84ad18e6c8bd418d159be210f5b","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5ad8ac14f20\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-qzkbcmm8mcxw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7edca57-ba27-4ad3-8604-bffa592f9a57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7edca57-ba27-4ad3-8604-bffa592f9a57\",\n      \"name\": \"d7edca57-ba27-4ad3-8604-bffa592f9a57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:47:17.981245925Z\",\n      \"endTime\": \"2026-10-07T11:16:24.854771823Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5ad8ac14f20\u003e: \n    failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-qzkbcmm8mcxw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7edca57-ba27-4ad3-8604-bffa592f9a57\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/d7edca57-ba27-4ad3-8604-bffa592f9a57\",\n      \"name\": \"d7edca57-ba27-4ad3-8604-bffa592f9a57\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:47:17.981245925Z\",\n      \"endTime\": \"2026-10-07T11:16:24.854771823Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"93b35e3e315079d9873edb07e592947139e68374e2357fdc619b6bbc7e26063e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"f5875557a07249abfea079a118855841355a791d06199b1a53afdd74eb9ed76c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-6n5rz7\" with version 4.21.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d92db70c300\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-6n5rz7: failed waiting for cluster=\"cluster-zstream-4-21-6n5rz7\" in resourcegroup=\"rg-zstream-upgrade-4-21-2tzcc6bmrz5d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c2c70dd-1f90-4d59-8d60-3f09ac473116\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c2c70dd-1f90-4d59-8d60-3f09ac473116\",\n      \"name\": \"9c2c70dd-1f90-4d59-8d60-3f09ac473116\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:52.836132625Z\",\n      \"endTime\": \"2026-10-07T11:13:54.943163158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-21-6n5rz7\" with version 4.21.35 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1d92db70c300\u003e: \n    failed to create HCP cluster cluster-zstream-4-21-6n5rz7: failed waiting for cluster=\"cluster-zstream-4-21-6n5rz7\" in resourcegroup=\"rg-zstream-upgrade-4-21-2tzcc6bmrz5d\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c2c70dd-1f90-4d59-8d60-3f09ac473116\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9c2c70dd-1f90-4d59-8d60-3f09ac473116\",\n      \"name\": \"9c2c70dd-1f90-4d59-8d60-3f09ac473116\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:52.836132625Z\",\n      \"endTime\": \"2026-10-07T11:13:54.943163158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"94ce18b842050cae368cfc4b518e8ab59e9b64ac1dbf8992bbf3a4991a3ac365","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"481b8b75a1b96ca5473b5e3428bf844583807c6c42f822464b4a107e600e55af","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c0664c8ea20\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vf9txtzbnpnn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/42419f25-0a2f-4391-9457-e8a913e2ad07\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/42419f25-0a2f-4391-9457-e8a913e2ad07\",\n      \"name\": \"42419f25-0a2f-4391-9457-e8a913e2ad07\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:42.287174236Z\",\n      \"endTime\": \"2026-10-07T12:10:44.727935031Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c0664c8ea20\u003e: \n    failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-vf9txtzbnpnn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/42419f25-0a2f-4391-9457-e8a913e2ad07\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/42419f25-0a2f-4391-9457-e8a913e2ad07\",\n      \"name\": \"42419f25-0a2f-4391-9457-e8a913e2ad07\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:42.287174236Z\",\n      \"endTime\": \"2026-10-07T12:10:44.727935031Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9544875f44c6f09e585ba48495ea649b5ce6ad68d7b7e584be2c4b2800221254","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"50ea19fbe8e7d37bc39003918df0f3db3330d2f50721b75d7305c5fee9e0c335","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd1ce0d4a820\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-j4vrdqt97r78\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f0d846d-be78-4858-ac71-4df870ff7135\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f0d846d-be78-4858-ac71-4df870ff7135\",\n      \"name\": \"0f0d846d-be78-4858-ac71-4df870ff7135\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:49:05.191507353Z\",\n      \"endTime\": \"2026-10-07T10:18:14.59940059Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:101]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd1ce0d4a820\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-j4vrdqt97r78\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f0d846d-be78-4858-ac71-4df870ff7135\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/0f0d846d-be78-4858-ac71-4df870ff7135\",\n      \"name\": \"0f0d846d-be78-4858-ac71-4df870ff7135\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:49:05.191507353Z\",\n      \"endTime\": \"2026-10-07T10:18:14.59940059Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9e50dd40d75873d265c9c90348caf71eec85b9503e26a88c7a0f6e2dc3bd790b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"be2d920233867be6ec757af08451d598249bd0bcdda5bba21b83e5200e3fff3b","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-gwxd67\" at install version 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d69a88ca960\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-gwxd67: failed waiting for cluster=\"cp-ystream-upgrade-5-1-gwxd67\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-sz9hmhntp9kj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f8ab362-9e28-46d5-b04d-5b54432b711b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f8ab362-9e28-46d5-b04d-5b54432b711b\",\n      \"name\": \"6f8ab362-9e28-46d5-b04d-5b54432b711b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:47:03.647392913Z\",\n      \"endTime\": \"2026-10-07T11:16:04.847770934Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-gwxd67\" at install version 5.0\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d69a88ca960\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-1-gwxd67: failed waiting for cluster=\"cp-ystream-upgrade-5-1-gwxd67\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-sz9hmhntp9kj\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f8ab362-9e28-46d5-b04d-5b54432b711b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/6f8ab362-9e28-46d5-b04d-5b54432b711b\",\n      \"name\": \"6f8ab362-9e28-46d5-b04d-5b54432b711b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:47:03.647392913Z\",\n      \"endTime\": \"2026-10-07T11:16:04.847770934Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a17919b1145a39c84ae2198ebb7549d15e8b7b29827addacdd0e3c2e0cfa569c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"81065ecc30a88dbe3690b63b71b1ab98da072a79a5e00eafdac69a5cf5e2e363","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-xl9nkn\" with version 4.22.16 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x168a94b865c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-xl9nkn: failed waiting for cluster=\"cluster-zstream-4-22-xl9nkn\" in resourcegroup=\"rg-zstream-upgrade-4-22-fhf2rq9bfzcg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7de41487-4bc1-4310-b1a9-cd8693231f1a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7de41487-4bc1-4310-b1a9-cd8693231f1a\",\n      \"name\": \"7de41487-4bc1-4310-b1a9-cd8693231f1a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:46.782260142Z\",\n      \"endTime\": \"2026-10-07T11:13:54.851175708Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-22-xl9nkn\" with version 4.22.16 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x168a94b865c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-22-xl9nkn: failed waiting for cluster=\"cluster-zstream-4-22-xl9nkn\" in resourcegroup=\"rg-zstream-upgrade-4-22-fhf2rq9bfzcg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7de41487-4bc1-4310-b1a9-cd8693231f1a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7de41487-4bc1-4310-b1a9-cd8693231f1a\",\n      \"name\": \"7de41487-4bc1-4310-b1a9-cd8693231f1a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:46.782260142Z\",\n      \"endTime\": \"2026-10-07T11:13:54.851175708Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a24c0e92572daeb644212023ee1e9738981540ba2d6f305665990ff7f090003e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"554cf56528558edb3b288f90dec934fc7fd2db28015f4ef2d61362bc08676deb","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1d4613307440\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-xccktj7qbpzs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1d4613307440\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-xccktj7qbpzs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a2ac948e9abdf5e39bf692fa3934edecc8d182e8ceaa928f81688bec39764648","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"44595e30de386c5eb9f145b02f8c2bb5d743f010820664d1bfb5aebca2d405ef","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-6bdqbx\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x197b95cac700\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-6bdqbx: failed waiting for cluster=\"cp-ystream-upgrade-5-0-6bdqbx\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-n9sd4zrjt4nl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ce7ba26b-badb-435f-89f2-c66478a228c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ce7ba26b-badb-435f-89f2-c66478a228c2\",\n      \"name\": \"ce7ba26b-badb-435f-89f2-c66478a228c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:46:45.340468133Z\",\n      \"endTime\": \"2026-10-07T11:15:54.827855172Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-6bdqbx\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0x197b95cac700\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-6bdqbx: failed waiting for cluster=\"cp-ystream-upgrade-5-0-6bdqbx\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-n9sd4zrjt4nl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ce7ba26b-badb-435f-89f2-c66478a228c2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/ce7ba26b-badb-435f-89f2-c66478a228c2\",\n      \"name\": \"ce7ba26b-badb-435f-89f2-c66478a228c2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:46:45.340468133Z\",\n      \"endTime\": \"2026-10-07T11:15:54.827855172Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a369538259d4b85c6ec36a979aa350c2913e9906f2abce0c4815b9f70a58ad14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"dd8473cf389dd6f494be575174dc277ffa211b0aa8afa70458bd3be0e7b014db","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22fee450ece0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-v6kzg6nzqfwv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-v6kzg6nzqfwv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22fee450ece0\u003e: \n    failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-v6kzg6nzqfwv\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-v6kzg6nzqfwv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a4b6e94b8dd6ec0086a479f41c6c809993af1388eb579298dc118a527be94b65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"222845611c5a37be3043cb1e73ff7a15f6703e7a0b75ca16f7f6d42711571715","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22fba5918600\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4lbxg4t4t5gh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c801c4e-0016-44b0-adb5-e97e7115ba1f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c801c4e-0016-44b0-adb5-e97e7115ba1f\",\n      \"name\": \"7c801c4e-0016-44b0-adb5-e97e7115ba1f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:50:15.185124964Z\",\n      \"endTime\": \"2026-10-07T11:19:24.850746241Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22fba5918600\u003e: \n    failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4lbxg4t4t5gh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c801c4e-0016-44b0-adb5-e97e7115ba1f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7c801c4e-0016-44b0-adb5-e97e7115ba1f\",\n      \"name\": \"7c801c4e-0016-44b0-adb5-e97e7115ba1f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:50:15.185124964Z\",\n      \"endTime\": \"2026-10-07T11:19:24.850746241Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"abfeadf94142ff38bd5af4f5f5a275a7e54bec074213d0fde2cb9dcf73b00745","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"76f0209f03661a65125740c9733c3d4a11d4682178d1f51fbb322ae760b9115c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-dzdf5z\" with version 5.0.0-rc.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ec1ed438600\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-dzdf5z: failed waiting for cluster=\"cluster-zstream-5-0-dzdf5z\" in resourcegroup=\"rg-zstream-upgrade-5-0-fvdn76jhpwwl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1857bc40-fec4-4c28-a63e-44dc6ba3733e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1857bc40-fec4-4c28-a63e-44dc6ba3733e\",\n      \"name\": \"1857bc40-fec4-4c28-a63e-44dc6ba3733e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:54.381402613Z\",\n      \"endTime\": \"2026-10-07T11:13:54.951616064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-5-0-dzdf5z\" with version 5.0.0-rc.4 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2ec1ed438600\u003e: \n    failed to create HCP cluster cluster-zstream-5-0-dzdf5z: failed waiting for cluster=\"cluster-zstream-5-0-dzdf5z\" in resourcegroup=\"rg-zstream-upgrade-5-0-fvdn76jhpwwl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1857bc40-fec4-4c28-a63e-44dc6ba3733e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1857bc40-fec4-4c28-a63e-44dc6ba3733e\",\n      \"name\": \"1857bc40-fec4-4c28-a63e-44dc6ba3733e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:54.381402613Z\",\n      \"endTime\": \"2026-10-07T11:13:54.951616064Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aeae8d3e931b65d7ec9448bfc594e9fee4b1982816d99d46bcc87546140916ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"3d70d948708128afed2253c50f9bf1c308631c17ba710f3522db7f18ef353d1c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-hstn44bjct9p/cluster-candidate-4-23-b4dmlz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d5329548a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-b4dmlz: failed waiting for cluster=\"cluster-candidate-4-23-b4dmlz\" in resourcegroup=\"rg-candidate-4-23-hstn44bjct9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3cf4f402-ca73-41e8-bc7a-b95fe95ddbb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3cf4f402-ca73-41e8-bc7a-b95fe95ddbb3\",\n      \"name\": \"3cf4f402-ca73-41e8-bc7a-b95fe95ddbb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:15.500215383Z\",\n      \"endTime\": \"2026-10-07T11:13:24.832957555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-hstn44bjct9p/cluster-candidate-4-23-b4dmlz should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3d5329548a0\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-b4dmlz: failed waiting for cluster=\"cluster-candidate-4-23-b4dmlz\" in resourcegroup=\"rg-candidate-4-23-hstn44bjct9p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3cf4f402-ca73-41e8-bc7a-b95fe95ddbb3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3cf4f402-ca73-41e8-bc7a-b95fe95ddbb3\",\n      \"name\": \"3cf4f402-ca73-41e8-bc7a-b95fe95ddbb3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:15.500215383Z\",\n      \"endTime\": \"2026-10-07T11:13:24.832957555Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b0bea8f605d50862403a0ed91fe9913a121d69ebd968cfdc62d61a918f2fdf9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"d049db17883d8e9b0b2cefc43813652d974c416c9ec136721194aeabfe76194d","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd878dbfc8e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-rtvvfx44j8ld\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-rtvvfx44j8ld/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xd878dbfc8e0\u003e: \n    failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-rtvvfx44j8ld\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-rtvvfx44j8ld/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bbf36526c326fe8d9475cd7e8d5d05279f6268d7eae222cf8cda055732400d2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"1882d241a3c9a7844d4783366ce6767db79330aaad9fc90ff0cbd2d6aa042b9e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cl6plzw4cxhg/cluster-candidate-4-20-ptf54w should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3294f0080980\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-ptf54w: failed waiting for cluster=\"cluster-candidate-4-20-ptf54w\" in resourcegroup=\"rg-candidate-4-20-cl6plzw4cxhg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/246ff8c9-d11b-4a89-906d-6858801ce1bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/246ff8c9-d11b-4a89-906d-6858801ce1bc\",\n      \"name\": \"246ff8c9-d11b-4a89-906d-6858801ce1bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:12.829533656Z\",\n      \"endTime\": \"2026-10-07T10:21:14.528988109Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-cl6plzw4cxhg/cluster-candidate-4-20-ptf54w should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3294f0080980\u003e: \n    failed to create HCP cluster cluster-candidate-4-20-ptf54w: failed waiting for cluster=\"cluster-candidate-4-20-ptf54w\" in resourcegroup=\"rg-candidate-4-20-cl6plzw4cxhg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/246ff8c9-d11b-4a89-906d-6858801ce1bc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/246ff8c9-d11b-4a89-906d-6858801ce1bc\",\n      \"name\": \"246ff8c9-d11b-4a89-906d-6858801ce1bc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:12.829533656Z\",\n      \"endTime\": \"2026-10-07T10:21:14.528988109Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c2f16f558bf7c49bd6e672a954c2c32b9d9cf8f5333ac83a4f3ea3f77d0fc4d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"aa6f57794e5ffec707db8f8f3922b0d34c04ee60936df21bade7d3f757093a83","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3723bf130ca0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-vdnbdccqmbl9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-vdnbdccqmbl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3723bf130ca0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-vdnbdccqmbl9\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-vdnbdccqmbl9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c41bf214e931235954945c8035a8deada8519f629847c23fef3ec47c394d4910","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"b3c1cc9d18ffd5943aa3e4739d61d99e679706d5964b07a3b5374e971e51cc88","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2669c8834500\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-hsv7wr49qksv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f18e0c20-bbd8-41c2-9e07-ac4d066319fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f18e0c20-bbd8-41c2-9e07-ac4d066319fa\",\n      \"name\": \"f18e0c20-bbd8-41c2-9e07-ac4d066319fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:56.465322587Z\",\n      \"endTime\": \"2026-10-07T10:18:04.504330062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:92]: failed to create HCP cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2669c8834500\u003e: \n    failed to create HCP cluster cx-rg-hcp-cluster: failed waiting for cluster=\"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-hsv7wr49qksv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f18e0c20-bbd8-41c2-9e07-ac4d066319fa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f18e0c20-bbd8-41c2-9e07-ac4d066319fa\",\n      \"name\": \"f18e0c20-bbd8-41c2-9e07-ac4d066319fa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:56.465322587Z\",\n      \"endTime\": \"2026-10-07T10:18:04.504330062Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c43f8e5cd8fc8d5a15b4b6691fe91c7f6debee0563d9b00adb14d0ef314fd2b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"b950484d8435e9ff010711daf067e8e3f2a694fad38348dda90d2c31aa435f5d","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x162888dc49e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-bp2mcv6f588h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a9746092-ea51-4e95-908b-677b0574f350\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a9746092-ea51-4e95-908b-677b0574f350\",\n      \"name\": \"a9746092-ea51-4e95-908b-677b0574f350\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:50:56.187197768Z\",\n      \"endTime\": \"2026-10-07T10:20:04.504909256Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0x162888dc49e0\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-bp2mcv6f588h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a9746092-ea51-4e95-908b-677b0574f350\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a9746092-ea51-4e95-908b-677b0574f350\",\n      \"name\": \"a9746092-ea51-4e95-908b-677b0574f350\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:50:56.187197768Z\",\n      \"endTime\": \"2026-10-07T10:20:04.504909256Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c785242c586def299913475ec45b9de77543c464b193af3f388066e80480abb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"e8b5cdeb0f741da0ec81fbee861c309ad3dc7a968fe5127637a89e53b5181949","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-wjlj9cf4zcnw/cluster-candidate-5-1-k5brcg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c0270464680\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-k5brcg: failed waiting for cluster=\"cluster-candidate-5-1-k5brcg\" in resourcegroup=\"rg-candidate-5-1-wjlj9cf4zcnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9f59e7e7-9129-4701-b675-75127fcdc2eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9f59e7e7-9129-4701-b675-75127fcdc2eb\",\n      \"name\": \"9f59e7e7-9129-4701-b675-75127fcdc2eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:26.380363687Z\",\n      \"endTime\": \"2026-10-07T11:13:34.962794383Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-wjlj9cf4zcnw/cluster-candidate-5-1-k5brcg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2c0270464680\u003e: \n    failed to create HCP cluster cluster-candidate-5-1-k5brcg: failed waiting for cluster=\"cluster-candidate-5-1-k5brcg\" in resourcegroup=\"rg-candidate-5-1-wjlj9cf4zcnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9f59e7e7-9129-4701-b675-75127fcdc2eb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/9f59e7e7-9129-4701-b675-75127fcdc2eb\",\n      \"name\": \"9f59e7e7-9129-4701-b675-75127fcdc2eb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:26.380363687Z\",\n      \"endTime\": \"2026-10-07T11:13:34.962794383Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c9fabb52db079545db500eec8b61f73727873c598147ffe55c6cb3da5aba1ef9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"677239a911859f072d4ad2996b92ed87c2c9b33ef1b746a6f2c25221940cd869","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fd68dad0ac0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-6mv68nffvl22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd123085-ce38-4a8b-8b84-938bfcfb8b88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd123085-ce38-4a8b-8b84-938bfcfb8b88\",\n      \"name\": \"cd123085-ce38-4a8b-8b84-938bfcfb8b88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:56.476908409Z\",\n      \"endTime\": \"2026-10-07T10:18:04.678303243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fd68dad0ac0\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-6mv68nffvl22\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd123085-ce38-4a8b-8b84-938bfcfb8b88\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/cd123085-ce38-4a8b-8b84-938bfcfb8b88\",\n      \"name\": \"cd123085-ce38-4a8b-8b84-938bfcfb8b88\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:56.476908409Z\",\n      \"endTime\": \"2026-10-07T10:18:04.678303243Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dd2b2e242eddd4ea8cfbd075ae15a918db4b1bb7b0d20839d37f7e0431acd931","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"cec1415504f739ba8bb2e47520a81df319f676a287c871c00a9ca5c0bf800bf0","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-f8hx5m with version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37308114340\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-f8hx5m: failed waiting for cluster=\"np-version-upgrade-cluster-f8hx5m\" in resourcegroup=\"rg-np-version-upgrade-f8hx5m-8jdlnlxvfr8j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f71f735-23a2-44d3-8bee-778955a7585f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f71f735-23a2-44d3-8bee-778955a7585f\",\n      \"name\": \"1f71f735-23a2-44d3-8bee-778955a7585f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:50.690197671Z\",\n      \"endTime\": \"2026-10-07T12:08:54.942764093Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-f8hx5m with version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37308114340\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-f8hx5m: failed waiting for cluster=\"np-version-upgrade-cluster-f8hx5m\" in resourcegroup=\"rg-np-version-upgrade-f8hx5m-8jdlnlxvfr8j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f71f735-23a2-44d3-8bee-778955a7585f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/1f71f735-23a2-44d3-8bee-778955a7585f\",\n      \"name\": \"1f71f735-23a2-44d3-8bee-778955a7585f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:39:50.690197671Z\",\n      \"endTime\": \"2026-10-07T12:08:54.942764093Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dd637039a089e9e1561569a9c15568c16b8470da515311cc3a274170e4a0e8c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"ff985ecaf6a3ec92db07f03052cef058d21f798d558e0c502a4afa2f5001396c","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a50070451c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-8fphbzx94qxr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/84804da6-d82b-4a54-acff-55763e6c56e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/84804da6-d82b-4a54-acff-55763e6c56e3\",\n      \"name\": \"84804da6-d82b-4a54-acff-55763e6c56e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:50:58.529983615Z\",\n      \"endTime\": \"2026-10-07T10:20:04.629346768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a50070451c0\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-8fphbzx94qxr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/84804da6-d82b-4a54-acff-55763e6c56e3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/84804da6-d82b-4a54-acff-55763e6c56e3\",\n      \"name\": \"84804da6-d82b-4a54-acff-55763e6c56e3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:50:58.529983615Z\",\n      \"endTime\": \"2026-10-07T10:20:04.629346768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e4777cd1555b7a8e6e4904041b2f7f157cda8fc99c17d611c623972ddfeef476","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"22d4e8be87148da939695d13e1247fdbef5bf1728991abf73a2144aa28fb21d0","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbae29765720\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-z5fwltqgzc4v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c5482cdd-de26-4821-85d9-d6c90378d1e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c5482cdd-de26-4821-85d9-d6c90378d1e9\",\n      \"name\": \"c5482cdd-de26-4821-85d9-d6c90378d1e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:08.573668314Z\",\n      \"endTime\": \"2026-10-07T10:21:14.503850093Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xbae29765720\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-z5fwltqgzc4v\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c5482cdd-de26-4821-85d9-d6c90378d1e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/c5482cdd-de26-4821-85d9-d6c90378d1e9\",\n      \"name\": \"c5482cdd-de26-4821-85d9-d6c90378d1e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:52:08.573668314Z\",\n      \"endTime\": \"2026-10-07T10:21:14.503850093Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eb0887fc6ca29d76d3e70b863bfb43eae3393f23448acfcf8e7fe89262172a0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"ebde694ea93828cb64d71fde4525690d02bffab2e1de046b2ef585cf0a3ac975","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"6a9c2523f9881cd28e189784f3244e2b462a82a81d6d904e71798161eadd2b44","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2fbb237581e0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kc94rh5nhxb8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kc94rh5nhxb8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2fbb237ab310\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0x2fbb237581e0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kc94rh5nhxb8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\n    {\n        msg: \"failed waiting for cluster=\\\"cluster-ver-4-20\\\" in resourcegroup=\\\"rg-cluster-back-version-kc94rh5nhxb8\\\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0x2fbb237ab310\u003e{\n                s: \"timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-kc94rh5nhxb8\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-kc94rh5nhxb8, error: context deadline exceeded"}},{"row_id":"efe2aec8e6bd178fe1f88f6871ad7675cc707ed27b7cfcd51af823ac0fca33da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"a031589172d1628bbf1aaa7c92834693d2ad406fbe2d747a22903fa0d686e2c9","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fbdfa206980\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7zh6fp8fgksk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5147c191-3d75-411b-b997-ea46e2d7cb21\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5147c191-3d75-411b-b997-ea46e2d7cb21\",\n      \"name\": \"5147c191-3d75-411b-b997-ea46e2d7cb21\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:07.233759701Z\",\n      \"endTime\": \"2026-10-07T12:10:14.721254823Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3fbdfa206980\u003e: \n    failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-7zh6fp8fgksk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5147c191-3d75-411b-b997-ea46e2d7cb21\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/5147c191-3d75-411b-b997-ea46e2d7cb21\",\n      \"name\": \"5147c191-3d75-411b-b997-ea46e2d7cb21\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:07.233759701Z\",\n      \"endTime\": \"2026-10-07T12:10:14.721254823Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f093f443a7cf429c70b585144fff7eeafe4e33e3e82d6f221d84f61820f0ded4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"353b34662f60e8180bd22425317cae7ac8cb19040c048446d6da34eb0770224d","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c820c72ab60\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-dlvtfxrd2j7s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a0b16c5f-b10f-427f-9f76-5f47695c8e2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a0b16c5f-b10f-427f-9f76-5f47695c8e2f\",\n      \"name\": \"a0b16c5f-b10f-427f-9f76-5f47695c8e2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:50.969969751Z\",\n      \"endTime\": \"2026-10-07T12:10:54.729673473Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_missing_identities.go:113]: failed to create HCP cluster \"missing-mi-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3c820c72ab60\u003e: \n    failed to create HCP cluster missing-mi-hcp-cluster: failed waiting for cluster=\"missing-mi-hcp-cluster\" in resourcegroup=\"delete-mi-rg-dlvtfxrd2j7s\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a0b16c5f-b10f-427f-9f76-5f47695c8e2f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a0b16c5f-b10f-427f-9f76-5f47695c8e2f\",\n      \"name\": \"a0b16c5f-b10f-427f-9f76-5f47695c8e2f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T11:41:50.969969751Z\",\n      \"endTime\": \"2026-10-07T12:10:54.729673473Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f310432731b91f5db817dead9f563437c1bdcee122e4a903d21d6976982f5851","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"3d2085352b10228c638ccaffe94249943e09d92267a10ea6e8a6f5acbc29dce2","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-nwj77t\" with version 4.20.40 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x275bdca84120\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-nwj77t: failed waiting for cluster=\"cluster-zstream-4-20-nwj77t\" in resourcegroup=\"rg-zstream-upgrade-4-20-szq75nd4jwdq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4a7a5069-7fe3-4404-a6ab-059ffdfd5c85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4a7a5069-7fe3-4404-a6ab-059ffdfd5c85\",\n      \"name\": \"4a7a5069-7fe3-4404-a6ab-059ffdfd5c85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:35.717450122Z\",\n      \"endTime\": \"2026-10-07T11:13:44.953502768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-20-nwj77t\" with version 4.20.40 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x275bdca84120\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-nwj77t: failed waiting for cluster=\"cluster-zstream-4-20-nwj77t\" in resourcegroup=\"rg-zstream-upgrade-4-20-szq75nd4jwdq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4a7a5069-7fe3-4404-a6ab-059ffdfd5c85\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/4a7a5069-7fe3-4404-a6ab-059ffdfd5c85\",\n      \"name\": \"4a7a5069-7fe3-4404-a6ab-059ffdfd5c85\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T10:44:35.717450122Z\",\n      \"endTime\": \"2026-10-07T11:13:44.953502768Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f61417832b01412a083af0eae8264c1cb2884b64af7cc2e226b7b8bdc5252907","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2107763785899970560","occurred_at":"2026-10-07T09:23:41Z","failed_at":"e2e","signature_id":"f952fed5c3234700f79a41bdd08cf3c1afc8a1f474d83944941014db01bd9d40","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13ce01f6d240\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lf5wbzx9gtnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7376de8f-61ce-423e-aa70-1db4f06f54df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7376de8f-61ce-423e-aa70-1db4f06f54df\",\n      \"name\": \"7376de8f-61ce-423e-aa70-1db4f06f54df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:54.383796786Z\",\n      \"endTime\": \"2026-10-07T10:17:54.50128014Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13ce01f6d240\u003e: \n    failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lf5wbzx9gtnw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7376de8f-61ce-423e-aa70-1db4f06f54df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/7376de8f-61ce-423e-aa70-1db4f06f54df\",\n      \"name\": \"7376de8f-61ce-423e-aa70-1db4f06f54df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T09:48:54.383796786Z\",\n      \"endTime\": \"2026-10-07T10:17:54.50128014Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceClusterStatus] \\u003cno_message\\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceClusterStatus] \u003cno_message\u003e; [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":60,"failure_pattern_ids":["120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","1db42ae850b6bab5c706bcc83402e0b0e260fafe5a51f45595da86c0eee22a53","311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","5b0ae0d0ae667e9dc6bc533372f5307238e86e3ce05774fdb71a26133357b522"],"matched_occurrences":60,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-07T02:00:56Z","started_at":"2026-10-07T02:00:56Z","completed_at":"2026-10-07T04:25:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-07T04:39:43.285091343Z","timing_metadata_checked_at":"2026-10-07T04:39:43.285091343Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-07T04:39:43.289761928Z","region_metadata_checked_at":"2026-10-07T04:39:43.289761928Z"},"failed_at":["e2e"],"failed_test_count":33,"occurrences":[{"row_id":"0a1e5dfc2a40fe1e1b106076d39d837cf48b4ae955d07fc1ffb75fcc5bf51280","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"fac4ce6bbb505620cdc94744fa17f94c3c1fa837421bd30a8671b97b8cca987e","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2191d5b1d3e0\u003e: \n    failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-5wqhb2\" resourcegroup=\"rg-np-skip-minor-5wqhb2-pcpb5hkjlxqk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-5wqhb2-pcpb5hkjlxqk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-5wqhb2/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:489]: failed to upgrade nodepool nps-420-422 from 4.20.41 to 4.22.17\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2191d5b1d3e0\u003e: \n    failed to start nodepool \"nps-420-422\" update in cluster \"np-skip-minor-5wqhb2\" resourcegroup=\"rg-np-skip-minor-5wqhb2-pcpb5hkjlxqk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-5wqhb2-pcpb5hkjlxqk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-5wqhb2/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"0e70da0eee08a94992621098f14ec5565df6cb37df46606628218ef9b42a8f99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"c29e28b214fa390074f60edb54d46b123550e1ba11cc9a9e14a2585e50f1d30d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5g7cxp\" for cluster \"cluster-candidate-4-22-5g7cxp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2051b5cf8420\u003e: \n    failed to create NodePool np-5g7cxp: failed waiting for nodepool=\"np-5g7cxp\" for cluster \"cluster-candidate-4-22-5g7cxp\" in resourcegroup=\"rg-candidate-4-22-4vkh858b4p5m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/2078827d-3863-4f50-abad-4b08b8a19530\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/2078827d-3863-4f50-abad-4b08b8a19530\",\n      \"name\": \"2078827d-3863-4f50-abad-4b08b8a19530\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:49:22.855228495Z\",\n      \"endTime\": \"2026-10-07T03:18:22.99461352Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-7nxzb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-4vkh858b4p5m-candidate-5g7cxp--managed/providers/Microsoft.Compute/virtualMachines/r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-7nxzb to exist\\nMachine r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-bdpdz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-4vkh858b4p5m-candidate-5g7cxp--managed/providers/Microsoft.Compute/virtualMachines/r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-bdpdz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-7nxzb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-4vkh858b4p5m-candidate-5g7cxp--managed/providers/Microsoft.Compute/virtualMachines/r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-7nxzb to exist\\nMachine r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-bdpdz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-4vkh858b4p5m-candidate-5g7cxp--managed/providers/Microsoft.Compute/virtualMachines/r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-bdpdz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5g7cxp\" for cluster \"cluster-candidate-4-22-5g7cxp\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2051b5cf8420\u003e: \n    failed to create NodePool np-5g7cxp: failed waiting for nodepool=\"np-5g7cxp\" for cluster \"cluster-candidate-4-22-5g7cxp\" in resourcegroup=\"rg-candidate-4-22-4vkh858b4p5m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/2078827d-3863-4f50-abad-4b08b8a19530\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/2078827d-3863-4f50-abad-4b08b8a19530\",\n      \"name\": \"2078827d-3863-4f50-abad-4b08b8a19530\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:49:22.855228495Z\",\n      \"endTime\": \"2026-10-07T03:18:22.99461352Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-7nxzb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-4vkh858b4p5m-candidate-5g7cxp--managed/providers/Microsoft.Compute/virtualMachines/r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-7nxzb to exist\\nMachine r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-bdpdz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-4vkh858b4p5m-candidate-5g7cxp--managed/providers/Microsoft.Compute/virtualMachines/r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-bdpdz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-7nxzb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-4vkh858b4p5m-candidate-5g7cxp--managed/providers/Microsoft.Compute/virtualMachines/r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-7nxzb to exist\\nMachine r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-bdpdz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-4vkh858b4p5m-candidate-5g7cxp--managed/providers/Microsoft.Compute/virtualMachines/r3p7j3d3k1v6n6z-np-5g7cxp-kkrh8-bdpdz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"170c90d00f7892e9120595714c699dcdf7429da1f4daaa06a973ed9dd09bcb89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"45693c845a913fd4511bed4e661c2d3c6bfa8a4cc05cb75085a24072104e134b","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x266375a9e030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rsx69mvsw2xq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/86fb5972-fae9-42a5-b2cc-cef72453d35a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/86fb5972-fae9-42a5-b2cc-cef72453d35a\",\n      \"name\": \"86fb5972-fae9-42a5-b2cc-cef72453d35a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:19:24.17651873Z\",\n      \"endTime\": \"2026-10-07T02:48:31.262778869Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-24jdj-5chf2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rsx69mvsw2xq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-24jdj-5chf2 to exist\\nMachine cilium-cluster-cilium-np-24jdj-dvggb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rsx69mvsw2xq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-24jdj-dvggb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-24jdj-5chf2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rsx69mvsw2xq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-24jdj-5chf2 to exist\\nMachine cilium-cluster-cilium-np-24jdj-dvggb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rsx69mvsw2xq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-24jdj-dvggb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:193]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x266375a9e030\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: client rate limiter Wait returned an error: context deadline exceeded\n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-rsx69mvsw2xq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/86fb5972-fae9-42a5-b2cc-cef72453d35a\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/86fb5972-fae9-42a5-b2cc-cef72453d35a\",\n      \"name\": \"86fb5972-fae9-42a5-b2cc-cef72453d35a\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:19:24.17651873Z\",\n      \"endTime\": \"2026-10-07T02:48:31.262778869Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine cilium-cluster-cilium-np-24jdj-5chf2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rsx69mvsw2xq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-24jdj-5chf2 to exist\\nMachine cilium-cluster-cilium-np-24jdj-dvggb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rsx69mvsw2xq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-24jdj-dvggb to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-24jdj-5chf2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rsx69mvsw2xq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-24jdj-5chf2 to exist\\nMachine cilium-cluster-cilium-np-24jdj-dvggb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-rsx69mvsw2xq--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-24jdj-dvggb to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"17307ee93b30185b3ea821eedae56fa734559a3470145347065f0dc888b9a13e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"dcfadc4ebeef20592def4e3d8be48fe15094c12bcb500f3186b13d44a79114cc","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b65283c5780\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qcwx6cjhvdwl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/75bd1e92-642d-44cc-9f1d-7a0b14555639\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/75bd1e92-642d-44cc-9f1d-7a0b14555639\",\n      \"name\": \"75bd1e92-642d-44cc-9f1d-7a0b14555639\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T03:30:42.032719201Z\",\n      \"endTime\": \"2026-10-07T03:59:47.578313034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g3y2u3u9b3i0a3e-np-delete-main-mk65h-96sxf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qcwx6cjhvdwl--managed/providers/Microsoft.Compute/virtualMachines/g3y2u3u9b3i0a3e-np-delete-main-mk65h-96sxf to exist\\nMachine g3y2u3u9b3i0a3e-np-delete-main-mk65h-hb6kz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qcwx6cjhvdwl--managed/providers/Microsoft.Compute/virtualMachines/g3y2u3u9b3i0a3e-np-delete-main-mk65h-hb6kz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g3y2u3u9b3i0a3e-np-delete-main-mk65h-96sxf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qcwx6cjhvdwl--managed/providers/Microsoft.Compute/virtualMachines/g3y2u3u9b3i0a3e-np-delete-main-mk65h-96sxf to exist\\nMachine g3y2u3u9b3i0a3e-np-delete-main-mk65h-hb6kz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qcwx6cjhvdwl--managed/providers/Microsoft.Compute/virtualMachines/g3y2u3u9b3i0a3e-np-delete-main-mk65h-hb6kz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3b65283c5780\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qcwx6cjhvdwl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/75bd1e92-642d-44cc-9f1d-7a0b14555639\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/75bd1e92-642d-44cc-9f1d-7a0b14555639\",\n      \"name\": \"75bd1e92-642d-44cc-9f1d-7a0b14555639\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T03:30:42.032719201Z\",\n      \"endTime\": \"2026-10-07T03:59:47.578313034Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g3y2u3u9b3i0a3e-np-delete-main-mk65h-96sxf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qcwx6cjhvdwl--managed/providers/Microsoft.Compute/virtualMachines/g3y2u3u9b3i0a3e-np-delete-main-mk65h-96sxf to exist\\nMachine g3y2u3u9b3i0a3e-np-delete-main-mk65h-hb6kz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qcwx6cjhvdwl--managed/providers/Microsoft.Compute/virtualMachines/g3y2u3u9b3i0a3e-np-delete-main-mk65h-hb6kz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g3y2u3u9b3i0a3e-np-delete-main-mk65h-96sxf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qcwx6cjhvdwl--managed/providers/Microsoft.Compute/virtualMachines/g3y2u3u9b3i0a3e-np-delete-main-mk65h-96sxf to exist\\nMachine g3y2u3u9b3i0a3e-np-delete-main-mk65h-hb6kz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-qcwx6cjhvdwl--managed/providers/Microsoft.Compute/virtualMachines/g3y2u3u9b3i0a3e-np-delete-main-mk65h-hb6kz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-6xhlmlzjgvst\" to finish creating: GET https://management"}},{"row_id":"1e9b068061d4db415afe977290b5bf2d90c7790bd14cbca86c1ed48165344b83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"5be1160c2fa84fb7c2ea3eb2b520a681bba0ca7197c4252afbb61a475fb1097d","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b54f9373160\u003e: \n    failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-rb5sd7\" resourcegroup=\"rg-np-noedge-rb5sd7-m8nqd7x28f72\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-rb5sd7-m8nqd7x28f72/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-rb5sd7/nodePools/npnoedge-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:349]: failed to upgrade nodepool npnoedge-4-20 from 4.20.40 to 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2b54f9373160\u003e: \n    failed to start nodepool \"npnoedge-4-20\" update in cluster \"np-noedge-rb5sd7\" resourcegroup=\"rg-np-noedge-rb5sd7-m8nqd7x28f72\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-rb5sd7-m8nqd7x28f72/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-rb5sd7/nodePools/npnoedge-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3630f3a3f552e4c40b0a9973a5607b455955723437430aec87fc184d125f80f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"f8f6ef862ba076aa00ce8d20e4b36cddbf5813529ac198d1dd60133ef734dda8","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2846e9590e40\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-6gjzds\" resourcegroup=\"rg-np-downgrade-6gjzds-86mbqsnm8drv\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-6gjzds-86mbqsnm8drv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-6gjzds/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:628]: failed to update nodepool npdg-4-21 to downgrade target 4.21.35\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2846e9590e40\u003e: \n    failed to start nodepool \"npdg-4-21\" update in cluster \"np-downgrade-6gjzds\" resourcegroup=\"rg-np-downgrade-6gjzds-86mbqsnm8drv\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-6gjzds-86mbqsnm8drv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-downgrade-6gjzds/nodePools/npdg-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"3b0a6260621ee3c39a7e541bf824bf721a0c6c5e5b9378c727c12252c2da9ee6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"0058e792d032ae53f5bca2e179ae1c409cdd5b3c2bb64768e1eb44b0951a5c10","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x231d4b99bc00\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-lxpkjd\" resourcegroup=\"rg-np-version-upgrade-lxpkjd-r2d8ctn9k9gz\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lxpkjd-r2d8ctn9k9gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-lxpkjd/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x231d4b99bc00\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-lxpkjd\" resourcegroup=\"rg-np-version-upgrade-lxpkjd-r2d8ctn9k9gz\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lxpkjd-r2d8ctn9k9gz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-lxpkjd/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4a08090c7d5c81f39dcf1820d5d0122b44b1035cf217cf2ba33a8ebc74bb37f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"f3a567f87a32f64da957df0785589b6bd3c47799e90deadbfe5bc1b927ed369a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8l7dvc\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3cfbe8935230\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-xjbz46kpx22f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-8l7dvc\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-22-8l7dvc\" to 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3cfbe8935230\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-xjbz46kpx22f/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-22-8l7dvc\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4a2b8f0a26f4ae02abfee438c9bdbfdc021c5250a1517d92dc0865697f474377","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"6ce961a4bc8897fbd7b98df3cc5a58955ed5e747b18252fdfef55c7377a098c4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-t6wdwh\" for cluster \"cluster-candidate-5-1-t6wdwh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3dd08fa06500\u003e: \n    failed to create NodePool np-t6wdwh: failed waiting for nodepool=\"np-t6wdwh\" for cluster \"cluster-candidate-5-1-t6wdwh\" in resourcegroup=\"rg-candidate-5-1-kfz288sb9rh2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1d57d7e0-5573-4196-8ff1-f505d1b163e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1d57d7e0-5573-4196-8ff1-f505d1b163e5\",\n      \"name\": \"1d57d7e0-5573-4196-8ff1-f505d1b163e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:52:22.077189924Z\",\n      \"endTime\": \"2026-10-07T03:21:23.162976544Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-6w44l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kfz288sb9rh2-candidate-t6wdwh--managed/providers/Microsoft.Compute/virtualMachines/u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-6w44l to exist\\nMachine u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-wd24r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kfz288sb9rh2-candidate-t6wdwh--managed/providers/Microsoft.Compute/virtualMachines/u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-wd24r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-6w44l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kfz288sb9rh2-candidate-t6wdwh--managed/providers/Microsoft.Compute/virtualMachines/u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-6w44l to exist\\nMachine u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-wd24r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kfz288sb9rh2-candidate-t6wdwh--managed/providers/Microsoft.Compute/virtualMachines/u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-wd24r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-t6wdwh\" for cluster \"cluster-candidate-5-1-t6wdwh\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3dd08fa06500\u003e: \n    failed to create NodePool np-t6wdwh: failed waiting for nodepool=\"np-t6wdwh\" for cluster \"cluster-candidate-5-1-t6wdwh\" in resourcegroup=\"rg-candidate-5-1-kfz288sb9rh2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1d57d7e0-5573-4196-8ff1-f505d1b163e5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/1d57d7e0-5573-4196-8ff1-f505d1b163e5\",\n      \"name\": \"1d57d7e0-5573-4196-8ff1-f505d1b163e5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:52:22.077189924Z\",\n      \"endTime\": \"2026-10-07T03:21:23.162976544Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-6w44l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kfz288sb9rh2-candidate-t6wdwh--managed/providers/Microsoft.Compute/virtualMachines/u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-6w44l to exist\\nMachine u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-wd24r: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kfz288sb9rh2-candidate-t6wdwh--managed/providers/Microsoft.Compute/virtualMachines/u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-wd24r to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-6w44l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kfz288sb9rh2-candidate-t6wdwh--managed/providers/Microsoft.Compute/virtualMachines/u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-6w44l to exist\\nMachine u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-wd24r: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-kfz288sb9rh2-candidate-t6wdwh--managed/providers/Microsoft.Compute/virtualMachines/u3e6h1a2m5i1o7s-np-t6wdwh-92gd4-wd24r to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4d0b9ba21dfde770d42c14f2fdfa0ddf399e537fe5ac9028d3a002545ed87ee5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"f88443bde247b05fc776f561d632239634a07c67d0f9534f9fadfe4591ff0ae7","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x316f2960af60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5wjtv7scf56q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0x316f2960af60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-5wjtv7scf56q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"61e7e8692173edd235fce55317d6b8e4605867470039c9463bbf74443618c5ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"4af35e4cc7af67899aa17ed379f7e70c8cb4b98d18356ff405fe1baaa2a79cc2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mfcgvm\" for cluster \"cluster-candidate-4-20-mfcgvm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5d26d82a420\u003e: \n    failed to create NodePool np-mfcgvm: failed waiting for nodepool=\"np-mfcgvm\" for cluster \"cluster-candidate-4-20-mfcgvm\" in resourcegroup=\"rg-candidate-4-20-lh5ndcqmnzxh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c538dcdd-16ed-410d-8083-51bdf151f95d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c538dcdd-16ed-410d-8083-51bdf151f95d\",\n      \"name\": \"c538dcdd-16ed-410d-8083-51bdf151f95d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:50:54.360465912Z\",\n      \"endTime\": \"2026-10-07T03:20:03.12214784Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-lkcwf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lh5ndcqmnzxh-candidate-mfcgvm--managed/providers/Microsoft.Compute/virtualMachines/g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-lkcwf to exist\\nMachine g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-qmwtz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lh5ndcqmnzxh-candidate-mfcgvm--managed/providers/Microsoft.Compute/virtualMachines/g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-qmwtz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-lkcwf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lh5ndcqmnzxh-candidate-mfcgvm--managed/providers/Microsoft.Compute/virtualMachines/g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-lkcwf to exist\\nMachine g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-qmwtz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lh5ndcqmnzxh-candidate-mfcgvm--managed/providers/Microsoft.Compute/virtualMachines/g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-qmwtz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-mfcgvm\" for cluster \"cluster-candidate-4-20-mfcgvm\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x5d26d82a420\u003e: \n    failed to create NodePool np-mfcgvm: failed waiting for nodepool=\"np-mfcgvm\" for cluster \"cluster-candidate-4-20-mfcgvm\" in resourcegroup=\"rg-candidate-4-20-lh5ndcqmnzxh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c538dcdd-16ed-410d-8083-51bdf151f95d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c538dcdd-16ed-410d-8083-51bdf151f95d\",\n      \"name\": \"c538dcdd-16ed-410d-8083-51bdf151f95d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:50:54.360465912Z\",\n      \"endTime\": \"2026-10-07T03:20:03.12214784Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-lkcwf: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lh5ndcqmnzxh-candidate-mfcgvm--managed/providers/Microsoft.Compute/virtualMachines/g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-lkcwf to exist\\nMachine g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-qmwtz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lh5ndcqmnzxh-candidate-mfcgvm--managed/providers/Microsoft.Compute/virtualMachines/g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-qmwtz to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-lkcwf: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lh5ndcqmnzxh-candidate-mfcgvm--managed/providers/Microsoft.Compute/virtualMachines/g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-lkcwf to exist\\nMachine g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-qmwtz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-lh5ndcqmnzxh-candidate-mfcgvm--managed/providers/Microsoft.Compute/virtualMachines/g5c8w8d3q7e5i9l-np-mfcgvm-64sl9-qmwtz to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"698a5e209ecae337bf5d0f6963e932324b642be50f730118448e65e2354e567c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"e68cdd216d0150edd847f603ef682998ad7b0271775710e0858f3f7f5b37530f","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37d6f372b6a0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-8sc8h7sjfdcv\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8sc8h7sjfdcv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:189]: failed to update AZ nodepool autoscale-az max replicas to 4\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37d6f372b6a0\u003e: \n    failed to start nodepool \"autoscale-az\" update in cluster \"np-autoscale-cluster\" resourcegroup=\"np-autoscaling-8sc8h7sjfdcv\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-8sc8h7sjfdcv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-autoscale-cluster/nodePools/autoscale-az\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6bb39dccba675f1845870fc93db99bd8763b744b4936d89113d48cf5460865b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"bbea7e2da871e62baa803b44294e1265be218d3de0dc6cf6f671b9fcb2dfd871","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-p4lgwf\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1b0947b50a20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-2gpxkvcjbvk8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-p4lgwf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-22-p4lgwf\" on minor version 4.22\nUnexpected error:\n    \u003c*exported.ResponseError | 0x1b0947b50a20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-2gpxkvcjbvk8/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-22-p4lgwf\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"758ec409aa4c14d71bb7eedb20dd08ad3dbeccdeb6aba7445bc125095f25e788","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"3f3489b7375ffb121a83750f4359bd32c3f7d88aea2c87381d44bdbba593595c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3dec911a2048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-54gvlbbq8fjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c7f98f33-b298-4d0a-a913-93aac3ffc091\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c7f98f33-b298-4d0a-a913-93aac3ffc091\",\n      \"name\": \"c7f98f33-b298-4d0a-a913-93aac3ffc091\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:20:55.959277742Z\",\n      \"endTime\": \"2026-10-07T02:50:00.936442552Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-d5fls-gl48s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-d5fls-m9lj5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-54gvlbbq8fjw--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-d5fls-m9lj5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-d5fls-gl48s: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-d5fls-m9lj5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-54gvlbbq8fjw--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-d5fls-m9lj5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x3dec911a2048\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-54gvlbbq8fjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c7f98f33-b298-4d0a-a913-93aac3ffc091\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/c7f98f33-b298-4d0a-a913-93aac3ffc091\",\n      \"name\": \"c7f98f33-b298-4d0a-a913-93aac3ffc091\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:20:55.959277742Z\",\n      \"endTime\": \"2026-10-07T02:50:00.936442552Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-d5fls-gl48s: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine agg-cluster-agg-np-a-d5fls-m9lj5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-54gvlbbq8fjw--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-d5fls-m9lj5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-d5fls-gl48s: Creating: virtualmachine creating or updating\\nMachine agg-cluster-agg-np-a-d5fls-m9lj5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-54gvlbbq8fjw--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-d5fls-m9lj5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"79fc31ded7a3702ab740a8241fd77ded2917020911034178e9b9c4f9f8363b5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"7f76fffd8b4b60d1d840bb53756f03a0b49f3555b816311695931c4a85b6abbc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-82pv4r\" for cluster \"cluster-candidate-4-23-82pv4r\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33dd9953ad20\u003e: \n    failed to create NodePool np-82pv4r: failed waiting for nodepool=\"np-82pv4r\" for cluster \"cluster-candidate-4-23-82pv4r\" in resourcegroup=\"rg-candidate-4-23-6g4s6hnh7twn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/763f07a9-7f84-4d35-b440-501db428cd35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/763f07a9-7f84-4d35-b440-501db428cd35\",\n      \"name\": \"763f07a9-7f84-4d35-b440-501db428cd35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:49:16.715244148Z\",\n      \"endTime\": \"2026-10-07T03:18:22.978126797Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-ffv96: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6g4s6hnh7twn-candidate-82pv4r--managed/providers/Microsoft.Compute/virtualMachines/b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-ffv96 to exist\\nMachine b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-nncb2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6g4s6hnh7twn-candidate-82pv4r--managed/providers/Microsoft.Compute/virtualMachines/b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-nncb2 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-ffv96: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6g4s6hnh7twn-candidate-82pv4r--managed/providers/Microsoft.Compute/virtualMachines/b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-ffv96 to exist\\nMachine b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-nncb2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6g4s6hnh7twn-candidate-82pv4r--managed/providers/Microsoft.Compute/virtualMachines/b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-nncb2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-82pv4r\" for cluster \"cluster-candidate-4-23-82pv4r\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x33dd9953ad20\u003e: \n    failed to create NodePool np-82pv4r: failed waiting for nodepool=\"np-82pv4r\" for cluster \"cluster-candidate-4-23-82pv4r\" in resourcegroup=\"rg-candidate-4-23-6g4s6hnh7twn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/763f07a9-7f84-4d35-b440-501db428cd35\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/763f07a9-7f84-4d35-b440-501db428cd35\",\n      \"name\": \"763f07a9-7f84-4d35-b440-501db428cd35\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:49:16.715244148Z\",\n      \"endTime\": \"2026-10-07T03:18:22.978126797Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-ffv96: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6g4s6hnh7twn-candidate-82pv4r--managed/providers/Microsoft.Compute/virtualMachines/b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-ffv96 to exist\\nMachine b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-nncb2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6g4s6hnh7twn-candidate-82pv4r--managed/providers/Microsoft.Compute/virtualMachines/b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-nncb2 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-ffv96: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6g4s6hnh7twn-candidate-82pv4r--managed/providers/Microsoft.Compute/virtualMachines/b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-ffv96 to exist\\nMachine b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-nncb2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-6g4s6hnh7twn-candidate-82pv4r--managed/providers/Microsoft.Compute/virtualMachines/b9q6c1i1o4c2m6f-np-82pv4r-tcwl6-nncb2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7a2be1530240dbade6c95afb9bed7e71c83639968c830f4d16a3ce2be3071340","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"b4c938ffd1e07317c7f1a8b66bdd6168fa2ea345151308900639a09f23bc84fd","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-wfqdx6\" on minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0x35670f110a20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-m89bbfgxxvxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-wfqdx6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-5-0-wfqdx6\" on minor version 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0x35670f110a20\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-m89bbfgxxvxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-5-0-wfqdx6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7d0445771d7fee562937d9890097aafea6389c67a4f378e1238baa9de9ae8e07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"eca13765fa9405f95252cf15d7ab0f204511f501fe8006308939ad15b76113e0","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-jq9x5k\" on minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x13bed9716540\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-lr4cgtdlbsdg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-jq9x5k\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-23-jq9x5k\" on minor version 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x13bed9716540\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-lr4cgtdlbsdg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-23-jq9x5k\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"87c1a94ddaebbd4c0e66a155b485c6cda070b8a003e8a7109b60c22321283b7b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"4cb77a8e2eb04e51d1e31b43fc33204e1adae88dabf9e8306d0b10cbba5bc76e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x17b599a4e4b0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rvt8dfwzw6cv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0x17b599a4e4b0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-rvt8dfwzw6cv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"88f6e902ae5608db7c684e7c0d746cb0de0c25f9aae4787915fe4a20c7da09c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"9338ca7a8b3338ded2c268d7f0328d1f40dc0524254e97f8e79312c850e87765","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-h5kqdt\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2688b23f290\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-dqv8pzksfg2w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-h5kqdt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-23-h5kqdt\" to 4.23\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2688b23f290\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-dqv8pzksfg2w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-23-h5kqdt\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8bffd41d396913405d42bd2ae374c22d4606ec0e514e62c9074ac74c30a6bba3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"4aadc6cfa9574f8e73cbcf281be4f06712a54bf21ebdbda8d42084548d72bf1f","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-gcjnnr\" on minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2862c3242150\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-97btfzzdk5qt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-gcjnnr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-20-gcjnnr\" on minor version 4.20\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2862c3242150\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-97btfzzdk5qt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-gcjnnr\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.z-stream-update-policy\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.z-stream-update-policy]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.z-stream-update-policy\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8cda144c7b259f43e94c154ad76c7bb25e15ec9239ac25d55a0d3144d2c1ced4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"4cec21a3bb1485fbbf2fb4c6307e0674539ae34b252d1fd61badf2ee1e7ee5b2","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7308776540\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-92bg56\" resourcegroup=\"rg-np-version-upgrade-92bg56-8frlm96hwhs2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-92bg56-8frlm96hwhs2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-92bg56/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x7308776540\u003e: \n    failed to start nodepool \"npupgrade-4-20\" update in cluster \"np-version-upgrade-cluster-92bg56\" resourcegroup=\"rg-np-version-upgrade-92bg56-8frlm96hwhs2\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-92bg56-8frlm96hwhs2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-92bg56/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"96d83612354e814f58a03fe0393823c859f3f2351cfa15270dcb542791b4a4ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"ab88004ed09b9059f4902caf266cb767ff949a7b11999ba22a96ad8310ee0f0d","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2730e9c9c180\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-4297wtnz7lkp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e9b6cfe3-4de4-41e6-9696-6b3171ff2374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e9b6cfe3-4de4-41e6-9696-6b3171ff2374\",\n      \"name\": \"e9b6cfe3-4de4-41e6-9696-6b3171ff2374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:18:16.717381704Z\",\n      \"endTime\": \"2026-10-07T02:47:20.922673Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-9ndmt-lqfsm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-4297wtnz7lkp--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-9ndmt-lqfsm to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-9ndmt-lqfsm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-4297wtnz7lkp--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-9ndmt-lqfsm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:159]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2730e9c9c180\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-4297wtnz7lkp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e9b6cfe3-4de4-41e6-9696-6b3171ff2374\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/e9b6cfe3-4de4-41e6-9696-6b3171ff2374\",\n      \"name\": \"e9b6cfe3-4de4-41e6-9696-6b3171ff2374\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:18:16.717381704Z\",\n      \"endTime\": \"2026-10-07T02:47:20.922673Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 1; node pool condition AllNodesHealthy is False: 1 of 1 machines are not healthy\\nMachine private-kas-np-1-9ndmt-lqfsm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-4297wtnz7lkp--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-9ndmt-lqfsm to exist\\n; node pool condition AllMachinesReady is False: 1 of 1 machines are not ready\\nMachine private-kas-np-1-9ndmt-lqfsm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-4297wtnz7lkp--managed/providers/Microsoft.Compute/virtualMachines/private-kas-np-1-9ndmt-lqfsm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a3f990ab730e797ad0baf55f9c297c176dac694a8b6d8e0e5a64169449a0cd75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"7577f160ea754982b08b2f90867551e7ee2355fdac02ed66e99f5880127d9f79","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x313b400003a0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mqrq8s\" resourcegroup=\"rg-np-dg-minor-mqrq8s-zhzsjsdzbsjk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mqrq8s-zhzsjsdzbsjk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mqrq8s/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.41\nUnexpected error:\n    \u003c*fmt.wrapError | 0x313b400003a0\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-mqrq8s\" resourcegroup=\"rg-np-dg-minor-mqrq8s-zhzsjsdzbsjk\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-mqrq8s-zhzsjsdzbsjk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-mqrq8s/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a8965e5c443daa5cca1639f87cf61ae402682bdd61242772cda75c7ddb8c0508","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"5d7b068f725ac38a9e9614dfdb837b0fb5f432769ef58cb2757748484e960a34","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-r6bz6s\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2ed2f27658c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-7jmjflmmkzbd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-r6bz6s\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-1-r6bz6s\" to 5.1\nUnexpected error:\n    \u003c*exported.ResponseError | 0x2ed2f27658c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-7jmjflmmkzbd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-1-r6bz6s\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a9975f17589480cfeba82882974cc00cf8db4605c10503b4fb82664cc11f09ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"7011579edb7da9f880b9ea345fd4c99a061064cbde4affb3e7853dd76feb67c1","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-fg9qp7\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0x27965fe99170\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-bd2blzzd24m7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-fg9qp7\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-4-21-fg9qp7\" to 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0x27965fe99170\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-bd2blzzd24m7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-4-21-fg9qp7\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"af8ec5ea75a14c26a7e810dc0d5587baca94a18d86b5d22abb36ce5243cd54a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"1ce8678d638300e56c4666b5590395913aef8694ae27b3df675cb7ad0f7b01b1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5sbtvz\" for cluster \"cluster-candidate-5-0-5sbtvz\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x238acb1486c0\u003e: \n    failed to create NodePool np-5sbtvz: failed waiting for nodepool=\"np-5sbtvz\" for cluster \"cluster-candidate-5-0-5sbtvz\" in resourcegroup=\"rg-candidate-5-0-vmsjtdc4dmc2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/94d7a272-7f93-42b4-87a7-e5e1f90fc4aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/94d7a272-7f93-42b4-87a7-e5e1f90fc4aa\",\n      \"name\": \"94d7a272-7f93-42b4-87a7-e5e1f90fc4aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:53:44.600418059Z\",\n      \"endTime\": \"2026-10-07T03:22:52.95122632Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-h6d5n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vmsjtdc4dmc2-candidate-5sbtvz--managed/providers/Microsoft.Compute/virtualMachines/c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-h6d5n to exist\\nMachine c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-vxnm5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vmsjtdc4dmc2-candidate-5sbtvz--managed/providers/Microsoft.Compute/virtualMachines/c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-vxnm5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-h6d5n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vmsjtdc4dmc2-candidate-5sbtvz--managed/providers/Microsoft.Compute/virtualMachines/c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-h6d5n to exist\\nMachine c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-vxnm5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vmsjtdc4dmc2-candidate-5sbtvz--managed/providers/Microsoft.Compute/virtualMachines/c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-vxnm5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5sbtvz\" for cluster \"cluster-candidate-5-0-5sbtvz\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x238acb1486c0\u003e: \n    failed to create NodePool np-5sbtvz: failed waiting for nodepool=\"np-5sbtvz\" for cluster \"cluster-candidate-5-0-5sbtvz\" in resourcegroup=\"rg-candidate-5-0-vmsjtdc4dmc2\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/94d7a272-7f93-42b4-87a7-e5e1f90fc4aa\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/94d7a272-7f93-42b4-87a7-e5e1f90fc4aa\",\n      \"name\": \"94d7a272-7f93-42b4-87a7-e5e1f90fc4aa\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:53:44.600418059Z\",\n      \"endTime\": \"2026-10-07T03:22:52.95122632Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-h6d5n: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vmsjtdc4dmc2-candidate-5sbtvz--managed/providers/Microsoft.Compute/virtualMachines/c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-h6d5n to exist\\nMachine c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-vxnm5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vmsjtdc4dmc2-candidate-5sbtvz--managed/providers/Microsoft.Compute/virtualMachines/c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-vxnm5 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-h6d5n: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vmsjtdc4dmc2-candidate-5sbtvz--managed/providers/Microsoft.Compute/virtualMachines/c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-h6d5n to exist\\nMachine c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-vxnm5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vmsjtdc4dmc2-candidate-5sbtvz--managed/providers/Microsoft.Compute/virtualMachines/c8c6d7q4q9v4l8v-np-5sbtvz-54sd7-vxnm5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c9e9df9c1a524b2b8800fd47b22815f37ac2e191169030c856056aceccd0714c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"3b196b8c142b7de81731bf3edd13cc8231edfe7f5ca4159cff7cc65b94f4ec03","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-rdvrsz\" to 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc9376a12f60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-q7mr47v6tpnm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-rdvrsz\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:167]: failed to trigger y-stream upgrade of cluster \"cp-ystream-upgrade-5-0-rdvrsz\" to 5.0\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc9376a12f60\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-q7mr47v6tpnm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cp-ystream-upgrade-5-0-rdvrsz\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ccec12b83354e796d2c49e122a2fc42a089773d2becd0fd24a07d646b2808b2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"1cbf26ee1e5ab30922ecc4509d85eefbb36e982c6ef1ac8f2183590c4796adf3","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-8t9fz9\" on minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xd4a08aa9740\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-5jggbjxvkqw6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-8t9fz9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:157]: failed to enable immediate z-stream updates for cluster \"cluster-zstream-4-21-8t9fz9\" on minor version 4.21\nUnexpected error:\n    \u003c*exported.ResponseError | 0xd4a08aa9740\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-5jggbjxvkqw6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-21-8t9fz9\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d4777d1540bd0de72ef016ed03d09d625f125e794889c525af7227d8946a61d7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"cf3da2b8d691239f5c5c38c7c621ba3e4f4593ae21d6761ff8aa4453720a3201","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3b23ae9e81b0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-s64nkbtbvr4v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0x3b23ae9e81b0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-s64nkbtbvr4v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.cluster.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.cluster.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.cluster.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d5d53be3b7b44dfb4152fcb424e02457d0c9cd25ce910477edb63799a6c49859","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"4116ee0556c26bd8833a3374c74c6a16c11f1c9d2170f9475869ca6784808061","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jxhjcb\" for cluster \"cluster-candidate-4-21-jxhjcb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1846fa80a900\u003e: \n    failed to create NodePool np-jxhjcb: failed waiting for nodepool=\"np-jxhjcb\" for cluster \"cluster-candidate-4-21-jxhjcb\" in resourcegroup=\"rg-candidate-4-21-8w5wwd2jrrxw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/fa63c7e2-d755-41d4-a99d-595192e3cb6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/fa63c7e2-d755-41d4-a99d-595192e3cb6e\",\n      \"name\": \"fa63c7e2-d755-41d4-a99d-595192e3cb6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:50:57.300613412Z\",\n      \"endTime\": \"2026-10-07T03:20:02.945126702Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-gnvwg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8w5wwd2jrrxw-candidate-jxhjcb--managed/providers/Microsoft.Compute/virtualMachines/x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-gnvwg to exist\\nMachine x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-khzns: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8w5wwd2jrrxw-candidate-jxhjcb--managed/providers/Microsoft.Compute/virtualMachines/x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-khzns to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-gnvwg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8w5wwd2jrrxw-candidate-jxhjcb--managed/providers/Microsoft.Compute/virtualMachines/x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-gnvwg to exist\\nMachine x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-khzns: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8w5wwd2jrrxw-candidate-jxhjcb--managed/providers/Microsoft.Compute/virtualMachines/x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-khzns to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-jxhjcb\" for cluster \"cluster-candidate-4-21-jxhjcb\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1846fa80a900\u003e: \n    failed to create NodePool np-jxhjcb: failed waiting for nodepool=\"np-jxhjcb\" for cluster \"cluster-candidate-4-21-jxhjcb\" in resourcegroup=\"rg-candidate-4-21-8w5wwd2jrrxw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/fa63c7e2-d755-41d4-a99d-595192e3cb6e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/fa63c7e2-d755-41d4-a99d-595192e3cb6e\",\n      \"name\": \"fa63c7e2-d755-41d4-a99d-595192e3cb6e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:50:57.300613412Z\",\n      \"endTime\": \"2026-10-07T03:20:02.945126702Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-gnvwg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8w5wwd2jrrxw-candidate-jxhjcb--managed/providers/Microsoft.Compute/virtualMachines/x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-gnvwg to exist\\nMachine x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-khzns: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8w5wwd2jrrxw-candidate-jxhjcb--managed/providers/Microsoft.Compute/virtualMachines/x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-khzns to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-gnvwg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8w5wwd2jrrxw-candidate-jxhjcb--managed/providers/Microsoft.Compute/virtualMachines/x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-gnvwg to exist\\nMachine x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-khzns: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-8w5wwd2jrrxw-candidate-jxhjcb--managed/providers/Microsoft.Compute/virtualMachines/x1s8c4w1t6a8s2s-np-jxhjcb-mqmzf-khzns to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e3e61dfd689796cda31074129a124f3c1fb16ab1035ff56a29df6e59aeb17705","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"e915fc8a853ff7f35cb4923c9b73fd715420c8f36391d017033ce5747e871487","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x516fee870a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-hlstwcddsl64\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d577455b-50b8-4e4b-85d5-72a517862832\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d577455b-50b8-4e4b-85d5-72a517862832\",\n      \"name\": \"d577455b-50b8-4e4b-85d5-72a517862832\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:22:43.989386912Z\",\n      \"endTime\": \"2026-10-07T02:51:50.953880817Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1n4n6o5d5c1t1h-np-1-j4gb2-mpds5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hlstwcddsl64--managed/providers/Microsoft.Compute/virtualMachines/l1n4n6o5d5c1t1h-np-1-j4gb2-mpds5 to exist\\nMachine l1n4n6o5d5c1t1h-np-1-j4gb2-w2c2d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hlstwcddsl64--managed/providers/Microsoft.Compute/virtualMachines/l1n4n6o5d5c1t1h-np-1-j4gb2-w2c2d to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1n4n6o5d5c1t1h-np-1-j4gb2-mpds5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hlstwcddsl64--managed/providers/Microsoft.Compute/virtualMachines/l1n4n6o5d5c1t1h-np-1-j4gb2-mpds5 to exist\\nMachine l1n4n6o5d5c1t1h-np-1-j4gb2-w2c2d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hlstwcddsl64--managed/providers/Microsoft.Compute/virtualMachines/l1n4n6o5d5c1t1h-np-1-j4gb2-w2c2d to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x516fee870a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-hlstwcddsl64\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d577455b-50b8-4e4b-85d5-72a517862832\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/australiaeast/hcpOperationStatuses/d577455b-50b8-4e4b-85d5-72a517862832\",\n      \"name\": \"d577455b-50b8-4e4b-85d5-72a517862832\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-07T02:22:43.989386912Z\",\n      \"endTime\": \"2026-10-07T02:51:50.953880817Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l1n4n6o5d5c1t1h-np-1-j4gb2-mpds5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hlstwcddsl64--managed/providers/Microsoft.Compute/virtualMachines/l1n4n6o5d5c1t1h-np-1-j4gb2-mpds5 to exist\\nMachine l1n4n6o5d5c1t1h-np-1-j4gb2-w2c2d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hlstwcddsl64--managed/providers/Microsoft.Compute/virtualMachines/l1n4n6o5d5c1t1h-np-1-j4gb2-w2c2d to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l1n4n6o5d5c1t1h-np-1-j4gb2-mpds5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hlstwcddsl64--managed/providers/Microsoft.Compute/virtualMachines/l1n4n6o5d5c1t1h-np-1-j4gb2-mpds5 to exist\\nMachine l1n4n6o5d5c1t1h-np-1-j4gb2-w2c2d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hlstwcddsl64--managed/providers/Microsoft.Compute/virtualMachines/l1n4n6o5d5c1t1h-np-1-j4gb2-w2c2d to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fef3ac61761e876ab43bdfe2a69d2504f6d776d5513234b2e779bfc413a6032a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"8696e2e81d9653753fa26ee12fc06824880c95acb852d196ef0af32ebd7fd0a6","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12b4a1bbd7e0\u003e: \n    failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-7k79kh\" resourcegroup=\"rg-np-version-upgrade-7k79kh-mbzxs6n4k6hw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7k79kh-mbzxs6n4k6hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7k79kh/nodePools/npupgrade-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-21 to version 4.21.36\nUnexpected error:\n    \u003c*fmt.wrapError | 0x12b4a1bbd7e0\u003e: \n    failed to start nodepool \"npupgrade-4-21\" update in cluster \"np-version-upgrade-cluster-7k79kh\" resourcegroup=\"rg-np-version-upgrade-7k79kh-mbzxs6n4k6hw\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-7k79kh-mbzxs6n4k6hw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-7k79kh/nodePools/npupgrade-4-21\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ff7ee6adc8c230a02eec34fc6d8d80be292edf968f07622040a60888ab297d50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2107652362557460480","occurred_at":"2026-10-07T02:00:56Z","failed_at":"e2e","signature_id":"8cf036c6672baf731fc796bc4ac50089b693a6a516c1216bf1d9e70f531eaf90","test_name":"Customer should be able to update node pool labels and taints","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e96d2061e60\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-tn2zc5z4vc6x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-tn2zc5z4vc6x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e96d2061e60\u003e: \n    failed to start nodepool \"np-1\" update in cluster \"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-tn2zc5z4vc6x\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-tn2zc5z4vc6x/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-np-labels-taints/nodePools/np-1\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"aro-hcp.experimental.nodepool.max-update-duration\\\": unrecognized experimental tag\",\n        \"target\": \"tags[aro-hcp.experimental.nodepool.max-update-duration]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"aro-hcp.experimental.nodepool.max-update-duration\": unrecognized experimental tag; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":33,"failure_pattern_ids":["07c2215ec14108b514ce9b5137d5877a0da12abd87b89f0cec97c4a2e16cd568","23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","a0da30a0c27612be508c28754a49542d8b5ee63f56c4d31579815a7791580f72","afb4bb7f79f1adc8b037a175530f7cd4154fc1b7d3239a4e6c6847fba115a743","f0ba1e9dc36caed432a3f50675eca554c5652d89bb1d0ea22bf1b341f298f610"],"matched_occurrences":33,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}