{"meta":{"date":"2026-09-24","timezone":"UTC","generated_at":"2026-10-31T08:34:28Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":5,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T23:38:15Z","started_at":"2026-09-24T23:38:15Z","completed_at":"2026-09-25T02:06:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T02:13:49.826804531Z","timing_metadata_checked_at":"2026-09-25T02:13:49.826804531Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T02:13:49.83017162Z","region_metadata_checked_at":"2026-09-25T02:13:49.83017162Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1b0ff4b6c8b977178a032ddfcbc1acbcb0e3181779e0a9617667df4f33a9312e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","failed_at":"e2e","signature_id":"5159cce0088044d8bc192dbe0cedf4bcbebc146393b1f4ddad44f3a9d431c657","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxjdxchm and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lthncjjflrt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc001150fb0\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxjdxchm and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lthncjjflrt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc001150fb0\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"58584bc179225602c7e7f656c99ebbf1f0d42a72dce42e2f0c64f125d3b2afcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","failed_at":"e2e","signature_id":"bd9b3b7f5ce7275bd4f32fb355609d9038fed94688be48bf618a3c2e0b9ca102","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088a9a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-6bxr9l\" in resourcegroup=\"rg-np-noedge-6bxr9l-s8q7zdxcxq2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n \"name\": \"2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T01:01:50.31869719Z\",\n \"endTime\": \"2026-09-25T01:20:50.832464334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088a9a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-6bxr9l\" in resourcegroup=\"rg-np-noedge-6bxr9l-s8q7zdxcxq2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n \"name\": \"2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T01:01:50.31869719Z\",\n \"endTime\": \"2026-09-25T01:20:50.832464334Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aae18a0e590dfdd9c57a2e6c035321350b1312718930d67aa57902962c067ae9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","failed_at":"e2e","signature_id":"a7e14597b86188a7681e872f2938a0cc446f731377f16c3c6f83c2cc272f9397","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ml64wq with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013c02a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-ml64wq: failed waiting for cluster=\"np-version-upgrade-cluster-ml64wq\" in resourcegroup=\"rg-np-version-upgrade-ml64wq-zkbzk7hvlvck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n \"name\": \"2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T00:48:56.872501334Z\",\n \"endTime\": \"2026-09-25T01:18:00.947451943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ml64wq with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0013c02a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-ml64wq: failed waiting for cluster=\"np-version-upgrade-cluster-ml64wq\" in resourcegroup=\"rg-np-version-upgrade-ml64wq-zkbzk7hvlvck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n \"name\": \"2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-25T00:48:56.872501334Z\",\n \"endTime\": \"2026-09-25T01:18:00.947451943Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T23:00:14Z","started_at":"2026-09-24T23:00:14Z","completed_at":"2026-09-25T01:13:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-25T01:21:08.413605009Z","timing_metadata_checked_at":"2026-09-25T01:21:08.413605009Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-25T01:21:08.416356701Z","region_metadata_checked_at":"2026-09-25T01:21:08.416356701Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"0c3895616f6dc162c055df7320c686c5e185463bb65e94c414b7edecb64de2ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"369b11784caf993a1266d4a2191af0129727f06deaa23b4c15d665538f183d95","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010a2930\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-grftwltzcrgn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0010a2930\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-grftwltzcrgn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1afd47edd9e3e715b81dc5cd5545f76b6508414423f3cce9bd5bfb00dce6e6da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"6d276974ce3c3d3fd4a182577c36d577de093c2c2e4513abb2edf58915543439","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-854hq8s8lwp6/cluster-nightly-5-1-npws6l should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed0ea0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-npws6l: failed waiting for cluster=\"cluster-nightly-5-1-npws6l\" in resourcegroup=\"rg-nightly-5-1-854hq8s8lwp6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n \"name\": \"5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:33:11.266527055Z\",\n \"endTime\": \"2026-09-25T00:02:18.550348976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-854hq8s8lwp6/cluster-nightly-5-1-npws6l should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ed0ea0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-npws6l: failed waiting for cluster=\"cluster-nightly-5-1-npws6l\" in resourcegroup=\"rg-nightly-5-1-854hq8s8lwp6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n \"name\": \"5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:33:11.266527055Z\",\n \"endTime\": \"2026-09-25T00:02:18.550348976Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3cdcd1af82e9b7b34504522c99c5b113af32b0f3e2c9414d5a5fc6c4f758bae3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"a0d8bdae7638bd5124c215d98805edbc939d87889da4c9f0c02457aecd6221df","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxdxtlkr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-zhq2pjd4clbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc00066d200\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxdxtlkr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-zhq2pjd4clbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc00066d200\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"40b3667cdb94b7d5f1cfceed4f77484ef5315c8f5328b3097d1e7a2182c256b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"d8fdb12d282c5253e72136a3836928bc9db7e3e73c7eec5cc70b96bb18cad7d3","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000c772d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000c772d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"4de00383da021ef5b4826627680ada7e1bce2aa923a9d2dcb7f05c697100a09c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"6e1813acc449a6535ee059b4dc15d561d04689edaf2ac0927414e732fa390dd3","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9eba0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ftgp6q\" in resourcegroup=\"rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ftgp6q/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d9eba0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ftgp6q\" in resourcegroup=\"rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ftgp6q/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5e391d3496716438731882154bb2a524129aef8413bff3eab814a49218ca3229","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"163ba41eff7851369f9490bc51639528b5ab45ae39f71f38a3bbb8b825ec7e1d","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0002b1230\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cp45sss7fz6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0002b1230\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cp45sss7fz6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"708afdd8eab4ba701a6e0c93260dbbaf6cea67a75aebeaf54c5fef7640f5729a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"6d281a26d8ac50b58188d06190db9e64800fa423362a34399d5e8db878ac7e06","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b24be0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qvrt9xzb9h7m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\",\n \"name\": \"05a99209-7724-468c-8d11-789afc14d5d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:58:35.632366587Z\",\n \"endTime\": \"2026-09-25T00:17:38.767686221Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b24be0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qvrt9xzb9h7m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\",\n \"name\": \"05a99209-7724-468c-8d11-789afc14d5d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T23:58:35.632366587Z\",\n \"endTime\": \"2026-09-25T00:17:38.767686221Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"7142eceeb6195e57f9a1bf07ee45245be5817a802c395762f4c6aa814b435ce9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"2c2448420f0585ced642c7a920d0fe02a87408a0b9d29f4ef0081d2ced376382","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c9720\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-r2rlkr\" resourcegroup=\"rg-np-dg-minor-r2rlkr-ngfwrz48zjxx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-r2rlkr-ngfwrz48zjxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-r2rlkr/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c9720\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-r2rlkr\" resourcegroup=\"rg-np-dg-minor-r2rlkr-ngfwrz48zjxx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-r2rlkr-ngfwrz48zjxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-r2rlkr/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"7e9b3b1b26c86a21fe4037fb14ec82d9c9c213f3bcf92529544718e0094a110d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"7e29c61c5ec9623a1d7631cc8d8c2bf09662bec493f2b1e0c6761a8e6df9e943","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c260e0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-6rqpmb\" in resourcegroup=\"rg-np-skip-minor-6rqpmb-k2c9cm8vc6c5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6rqpmb-k2c9cm8vc6c5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6rqpmb/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c260e0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-6rqpmb\" in resourcegroup=\"rg-np-skip-minor-6rqpmb-k2c9cm8vc6c5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-6rqpmb-k2c9cm8vc6c5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-6rqpmb/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a135f52e8ea105138aaa09909dc685e51e13fbf9b4ba3e58022d4894fa29f74c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"62770c4f0e621d8962a43577db753baffbfa217cfa5c51a5444cc20ba8dcbaa1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a140\u003e: \n failed to create NodePool np-d25jl2: failed starting nodepool creation \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\" in resourcegroup=\"rg-nightly-4-20-kl7rft6vttsr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kl7rft6vttsr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-d25jl2/nodePools/np-d25jl2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0a140\u003e: \n failed to create NodePool np-d25jl2: failed starting nodepool creation \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\" in resourcegroup=\"rg-nightly-4-20-kl7rft6vttsr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kl7rft6vttsr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-d25jl2/nodePools/np-d25jl2\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d264ca2fd85e28f65b379c17478ce72e36c12d53b5728cbe6910fc297fae899f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"78618845cda7ac86dc1a7ff167f19e01de1f527c7ec62a7cbacf618e8f8a10e3","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013b2150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8g8c7wprtmtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013b2150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8g8c7wprtmtf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d88a65f42998d46f6c324da34de71246c797b1ad20f813feeee2115f77124c9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"0856a5b7612ceaf27aef57335208fdb96fec3578c31cded0af090414cbc459ac","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017055c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vj7bqc7l746k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0017055c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vj7bqc7l746k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"eeff63182d185fe43e5083fda604b8ba1da7b633b447b86d6747aaf2ef46d986","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","failed_at":"e2e","signature_id":"39cc51f3ed9337761bd1c79d82ca52214f6889badaaab5d696c36fd31637d8b2","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002802840\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qrllhzxjl5qf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002802840\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qrllhzxjl5qf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T20:45:24Z","started_at":"2026-09-24T20:45:24Z","completed_at":"2026-09-24T23:11:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T23:25:07.242246468Z","timing_metadata_checked_at":"2026-09-24T23:25:07.242246468Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T23:25:07.250814741Z","region_metadata_checked_at":"2026-09-24T23:25:07.250814741Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0b31300fdb25b8d6c1f8d1696189dfe589636f2b049cecc3b198d2f0b01315ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","occurred_at":"2026-09-24T20:45:24Z","failed_at":"e2e","signature_id":"3112eb82261054cccf2006078302df6fc47e6d756350b2f95e0fd2f5e3283de9","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011137a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vhxnnlnn4h55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n \"name\": \"8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:58:20.871989951Z\",\n \"endTime\": \"2026-09-24T21:17:23.599073745Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011137a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vhxnnlnn4h55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n \"name\": \"8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T20:58:20.871989951Z\",\n \"endTime\": \"2026-09-24T21:17:23.599073745Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fc961940d64263efffeba073e6c91f29bf52aeac6b8b6aad4a4450c7203a5ecf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","occurred_at":"2026-09-24T20:45:24Z","failed_at":"e2e","signature_id":"5c46d71a0a4a0451e999a028441816085c5b759ae39c7276fb744cd76b6b57a1","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxj5dtdr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-v7bfrx7ddjpw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000e01080\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxj5dtdr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-v7bfrx7ddjpw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc000e01080\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T17:14:30Z","started_at":"2026-09-24T17:14:30Z","completed_at":"2026-09-24T19:59:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T20:06:14.429368554Z","timing_metadata_checked_at":"2026-09-24T20:06:14.429368554Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T20:06:14.43232214Z","region_metadata_checked_at":"2026-09-24T20:06:14.43232214Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"98cd2c547e82228193888e5ee5ab46b117d3cfc6e888c0eff38566aa9594daa4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","occurred_at":"2026-09-24T17:14:30Z","failed_at":"e2e","signature_id":"849d9e776f46a941d1482cb6eef9a594d740471e89a56ba737d225b1ff968668","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxbkcvlx and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-q8hblx6hxzvd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0009ce940\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxbkcvlx and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-q8hblx6hxzvd/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0009ce940\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}},{"row_id":"c302cf72d9658711c38cd1395ef7709026a3bbc9e4ae9c2ec83bbd5bca32d208","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","occurred_at":"2026-09-24T17:14:30Z","failed_at":"e2e","signature_id":"0b0108efd360078eb545d9c8493397d07692133492d39ba4345c2659c856781d","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\nUnexpected error:\n \u003c*errors.StatusError | 0xc00103ab40\u003e: \n Operation cannot be fulfilled on nodes \"a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\nUnexpected error:\n \u003c*errors.StatusError | 0xc00103ab40\u003e: \n Operation cannot be fulfilled on nodes \"a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1c2f7c1ada887c9974dadc757d0c5e8d6fb57702022a1a3603efe040da29e12d","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["1c2f7c1ada887c9974dadc757d0c5e8d6fb57702022a1a3603efe040da29e12d","a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-24T02:00:13Z","started_at":"2026-09-24T02:00:13Z","completed_at":"2026-09-24T04:01:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-24T04:09:52.895699799Z","timing_metadata_checked_at":"2026-09-24T04:09:52.895699799Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-24T04:09:52.899371787Z","region_metadata_checked_at":"2026-09-24T04:09:52.899371787Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"091175b13bb91cb6d967c682b864bc3411362bfe3a2d716a98cf5c8ac9e70a4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"e328078175fb41c8dfbd4578e31c90a7c72998a5382576bff48a3e7cd16f791f","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010aab40\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jh96j8fnch7m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jh96j8fnch7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010aab40\u003e: \n failed to create HCP cluster v20260901: failed starting cluster creation \"v20260901\" in resourcegroup=\"v20260901preview-jh96j8fnch7m\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-jh96j8fnch7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/v20260901\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"163fc4b606445f2d4999df0fc14cdc847a108b9259ce6a0cf19ffd1a9960eb24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"0f42ca5bdd5424ebd462017e995420f979246e19a7f68c1a45f2e182770b37a3","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab7ea0\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-r6szljgv5m4p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-r6szljgv5m4p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ab7ea0\u003e: \n failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-r6szljgv5m4p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-r6szljgv5m4p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"466be3bdfe781bd637f4f8b20ae2c0e7b28abd1d0d2405cdf43972e33a853ec2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"f1db6949bdfa0c436485b175eee7c33e360c184472692055a39f14766a6685ef","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c0220\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-jrbwgvcxzd5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-jrbwgvcxzd5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:90]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009c0220\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed starting cluster creation \"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-jrbwgvcxzd5v\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-jrbwgvcxzd5v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cordon-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"767d7856e6472a8fb173fd73bc2c0d883c63dfdfa87eea6855fee532d2ed82ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"ae9cfa1cabd766fc0ba51cde7820c4aff0d9b5fe7146c215e79849948da08c5d","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001321160\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l2xrb7ls9tkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n \"name\": \"4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T03:02:54.530721397Z\",\n \"endTime\": \"2026-09-24T03:21:54.293711104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001321160\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l2xrb7ls9tkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n \"name\": \"4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-24T03:02:54.530721397Z\",\n \"endTime\": \"2026-09-24T03:21:54.293711104Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l2xrb7ls9tkm\" to finish creating: GET https://management"}},{"row_id":"b7f1906845926f7ab212ae6fc68fcaa2b34d6c0f2c70213a5aab35daf5aaa17d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"dfda133e06b8ddc58347c148dd004db522d35f830c18a3e53b5d3cde82d08ea0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005a06e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-vzmb26shnztz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vzmb26shnztz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005a06e0\u003e: \n failed to create HCP cluster ingress-cert-le: failed starting cluster creation \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-vzmb26shnztz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-vzmb26shnztz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ingress-cert-le\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"be4eec1afabd97970839b71b2b484775035a6e571f372184b3fc3f2a9d672405","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"3c9ac5911363cbb207f63f2d230928ca57009144d5f790aadca90edd77cd8e2e","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90d80\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pc4xbw2wvdgz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pc4xbw2wvdgz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e90d80\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pc4xbw2wvdgz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pc4xbw2wvdgz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"c55920c7c29064d9a86c53f29121e513a344361b9419f16b16d10b2eed97babb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"a043abfb4a137ba5c7fb99a1284be7cfe198204b214476f6fd57e00fee845d3a","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015fdec0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gk9hkpv4f4bq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015fdec0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gk9hkpv4f4bq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"fd7732f1b2077822f018d14a789f936569ad1abdf47c8ce1c5404b10c2d276e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","failed_at":"e2e","signature_id":"fa688a48213278aa49f8f4617b7ea8d841d18133d8772e5ee0c28401f6d5f777","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12d20\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fttxlklms5k2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-fttxlklms5k2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12d20\u003e: \n failed starting cluster creation \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-fttxlklms5k2\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-fttxlklms5k2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/fips-enabled-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Required value: field cannot be null\",\n \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]}]}