{"meta":{"date":"2026-09-29","timezone":"UTC","generated_at":"2026-10-14T14:28:25Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":107,"failed_runs":88,"runs_with_occurrences":88,"runs_with_failure_pattern_matches":83,"runs_with_unmatched_occurrences":5,"runs_without_occurrence_rows":19,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7251/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105068018659758080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7251,"pr_state":"closed","pr_sha":"766e0ef25d26bc8fad353f644663787115a38b6f","final_merged_sha":"766e0ef25d26bc8fad353f644663787115a38b6f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T22:51:40Z","started_at":"2026-09-29T22:51:40Z","completed_at":"2026-09-30T01:04:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T01:15:13.830511636Z","timing_metadata_checked_at":"2026-09-30T01:15:13.830511636Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T01:15:13.833753326Z","region_metadata_checked_at":"2026-09-30T01:15:13.833753326Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7255/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105067427300642816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7255,"pr_state":"closed","pr_sha":"37f4878cff7f4f8b4ba68bb5115008bea583098b","final_merged_sha":"37f4878cff7f4f8b4ba68bb5115008bea583098b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T22:49:19Z","started_at":"2026-09-29T22:49:19Z","completed_at":"2026-09-30T01:09:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T01:25:46.752422494Z","timing_metadata_checked_at":"2026-09-30T01:25:46.752422494Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T01:25:46.757945779Z","region_metadata_checked_at":"2026-09-30T01:25:46.757945779Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105060628308692992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T22:22:18Z","started_at":"2026-09-29T22:22:18Z","completed_at":"2026-09-30T00:37:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:53:48.301240595Z","timing_metadata_checked_at":"2026-09-30T00:53:48.301240595Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:53:48.304885784Z","region_metadata_checked_at":"2026-09-30T00:53:48.304885784Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"adf27b9aef850304c4d99a4abcfbc16defb6022af25392ce0e7e1f7c1a604b9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105060628308692992","occurred_at":"2026-09-29T22:22:18Z","failed_at":"e2e","signature_id":"d7100381ca39f41d29e0836b313aaf94a6b2d8d619220b2d8d452ecce9aa7dcc","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2a520\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-ghf4jfbdlg72\" to finish creating: GET https://rp.j8692992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/596355a9-5b86-46b1-9839-13786177e8e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/596355a9-5b86-46b1-9839-13786177e8e4\",\n \"name\": \"596355a9-5b86-46b1-9839-13786177e8e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:06:53.563704915Z\",\n \"endTime\": \"2026-09-29T23:35:57.862623465Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 21m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c2a520\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-ghf4jfbdlg72\" to finish creating: GET https://rp.j8692992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/596355a9-5b86-46b1-9839-13786177e8e4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/596355a9-5b86-46b1-9839-13786177e8e4\",\n \"name\": \"596355a9-5b86-46b1-9839-13786177e8e4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:06:53.563704915Z\",\n \"endTime\": \"2026-09-29T23:35:57.862623465Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 21m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f540b9557691364b919d0189e9d57dfc50fa3c4976a661a652a2980dab60b4fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105060628308692992","occurred_at":"2026-09-29T22:22:18Z","failed_at":"e2e","signature_id":"b84eb183fc30814489de66a12eaebf31c2e11f285d5fa67c28a5b4885017f2e3","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00114aa60\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-jglxzzbsxnh6\" to finish creating: GET https://rp.j8692992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ba10d5a-1e61-4b26-8ed8-33a90f52a770\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ba10d5a-1e61-4b26-8ed8-33a90f52a770\",\n \"name\": \"2ba10d5a-1e61-4b26-8ed8-33a90f52a770\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:06:32.779086006Z\",\n \"endTime\": \"2026-09-29T23:35:37.525993022Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 20m35s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00114aa60\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-jglxzzbsxnh6\" to finish creating: GET https://rp.j8692992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ba10d5a-1e61-4b26-8ed8-33a90f52a770\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ba10d5a-1e61-4b26-8ed8-33a90f52a770\",\n \"name\": \"2ba10d5a-1e61-4b26-8ed8-33a90f52a770\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:06:32.779086006Z\",\n \"endTime\": \"2026-09-29T23:35:37.525993022Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 20m35s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105055984161394688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7229,"pr_state":"closed","pr_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","final_merged_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T22:03:51Z","started_at":"2026-09-29T22:03:51Z","completed_at":"2026-09-30T00:35:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:42:52.150086653Z","timing_metadata_checked_at":"2026-09-30T00:42:52.150086653Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:42:52.15425014Z","region_metadata_checked_at":"2026-09-30T00:42:52.15425014Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"63762a17df525e11421a5d32ad090dd0609d9bb4cbd3f1adc1887342f61602ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105055984161394688","occurred_at":"2026-09-29T22:03:51Z","failed_at":"e2e","signature_id":"e3a48832d36b5100d3264e592c22aa50b936a7c4c8f7ef5634e6ff9cb089dbcd","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000eecfc0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-f25q6rmztl9d\" to finish creating: GET https://rp.j1394688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d3c2e124-eab6-4afe-a82b-d6f4e1f8cabf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d3c2e124-eab6-4afe-a82b-d6f4e1f8cabf\",\n \"name\": \"d3c2e124-eab6-4afe-a82b-d6f4e1f8cabf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:21:23.268055179Z\",\n \"endTime\": \"2026-09-29T23:40:26.327494387Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m8s8k4n5s9t3q7q-np-delete-main-b4h85-fm9gd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f25q6rmztl9d--managed/providers/Microsoft.Compute/virtualMachines/m8s8k4n5s9t3q7q-np-delete-main-b4h85-fm9gd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m8s8k4n5s9t3q7q-np-delete-main-b4h85-fm9gd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f25q6rmztl9d--managed/providers/Microsoft.Compute/virtualMachines/m8s8k4n5s9t3q7q-np-delete-main-b4h85-fm9gd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000eecfc0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-f25q6rmztl9d\" to finish creating: GET https://rp.j1394688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d3c2e124-eab6-4afe-a82b-d6f4e1f8cabf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d3c2e124-eab6-4afe-a82b-d6f4e1f8cabf\",\n \"name\": \"d3c2e124-eab6-4afe-a82b-d6f4e1f8cabf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:21:23.268055179Z\",\n \"endTime\": \"2026-09-29T23:40:26.327494387Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m8s8k4n5s9t3q7q-np-delete-main-b4h85-fm9gd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f25q6rmztl9d--managed/providers/Microsoft.Compute/virtualMachines/m8s8k4n5s9t3q7q-np-delete-main-b4h85-fm9gd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m8s8k4n5s9t3q7q-np-delete-main-b4h85-fm9gd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-f25q6rmztl9d--managed/providers/Microsoft.Compute/virtualMachines/m8s8k4n5s9t3q7q-np-delete-main-b4h85-fm9gd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-mvqbmn55cv5b\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105055507696848896","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7254,"pr_state":"closed","pr_sha":"fed46bdb633e2219c0b9e04a9984e1a83a49fd45","final_merged_sha":"fed46bdb633e2219c0b9e04a9984e1a83a49fd45","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T22:01:57Z","started_at":"2026-09-29T22:01:57Z","completed_at":"2026-09-30T00:24:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:32:16.369765136Z","timing_metadata_checked_at":"2026-09-30T00:32:16.369765136Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:32:16.372834826Z","region_metadata_checked_at":"2026-09-30T00:32:16.372834826Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"03ed1dcfe29dd3eafc2cfe5a913cce36040858655a118b72264fa214abe95c21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105055507696848896","occurred_at":"2026-09-29T22:01:57Z","failed_at":"alert","signature_id":"43e8e941070e87e545b7312032a0e11b759ff5fa2b2a787487c3a9652db3e71e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T23:07:08Z\n Ended: 2026-09-29T23:34:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6848896-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4v70gg0dc77jh6i9l6rr93v81st7vh-k2r6k8a7f3k5y2e\", pod=\"router-5c4bbd5cf4-5bqs8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4v70gg0dc77jh6i9l6rr93v81st7vh-k2r6k8a7f3k5y2e/router-5c4bbd5cf4-5bqs8 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T23:07:08Z\n Ended: 2026-09-29T23:34:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6848896-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4v6v9k2f4p2jhh963s28dni94k6h3d-n7f2l3l7q3v7r9c\", pod=\"router-68d995764-szrlt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4v6v9k2f4p2jhh963s28dni94k6h3d-n7f2l3l7q3v7r9c/router-68d995764-szrlt has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"39b96f5d4d455d813176fb93ee1129cd4725ac4811a95f9b31d00d2f9348c186","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105055507696848896","occurred_at":"2026-09-29T22:01:57Z","failed_at":"e2e","signature_id":"5122926c5ade8fbc736be83aa3f8fb88999a12813b939d32ff896605ad8168da","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fdc1a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-s7n5pwj29kws\" to finish creating: GET https://rp.j6848896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74696105-f579-4b73-9d4f-01cd26fb675e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74696105-f579-4b73-9d4f-01cd26fb675e\",\n \"name\": \"74696105-f579-4b73-9d4f-01cd26fb675e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T22:53:48.685100711Z\",\n \"endTime\": \"2026-09-29T23:22:58.576047333Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fdc1a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-s7n5pwj29kws\" to finish creating: GET https://rp.j6848896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74696105-f579-4b73-9d4f-01cd26fb675e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/74696105-f579-4b73-9d4f-01cd26fb675e\",\n \"name\": \"74696105-f579-4b73-9d4f-01cd26fb675e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T22:53:48.685100711Z\",\n \"endTime\": \"2026-09-29T23:22:58.576047333Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c477416fee9d6d3f978dd43b2520913e29315c46de034d434776a22cf6a5f02b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7254/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105055507696848896","occurred_at":"2026-09-29T22:01:57Z","failed_at":"e2e","signature_id":"dc00a4d7bc470559b6bed58c38ff280137de80a9b489b5ca06cdbd5e40d6fce4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-9xgkkc2gng7k/cluster-candidate-4-22-5cphq5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a567c0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-5cphq5: failed waiting for cluster=\"cluster-candidate-4-22-5cphq5\" in resourcegroup=\"rg-candidate-4-22-9xgkkc2gng7k\" to finish creating: GET https://rp.j6848896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/22da9982-475d-41a8-a97e-3a1a5b673130\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/22da9982-475d-41a8-a97e-3a1a5b673130\",\n \"name\": \"22da9982-475d-41a8-a97e-3a1a5b673130\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T22:53:52.362476002Z\",\n \"endTime\": \"2026-09-29T23:22:59.214530315Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 21m43s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-9xgkkc2gng7k/cluster-candidate-4-22-5cphq5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a567c0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-5cphq5: failed waiting for cluster=\"cluster-candidate-4-22-5cphq5\" in resourcegroup=\"rg-candidate-4-22-9xgkkc2gng7k\" to finish creating: GET https://rp.j6848896.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/22da9982-475d-41a8-a97e-3a1a5b673130\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/22da9982-475d-41a8-a97e-3a1a5b673130\",\n \"name\": \"22da9982-475d-41a8-a97e-3a1a5b673130\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T22:53:52.362476002Z\",\n \"endTime\": \"2026-09-29T23:22:59.214530315Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 21m43s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052569519788032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"e7500a3658c1692b6c32705cb7edf403b704aae4","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T21:50:17Z","started_at":"2026-09-29T21:50:17Z","completed_at":"2026-09-30T00:27:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:42:51.841948214Z","timing_metadata_checked_at":"2026-09-30T00:42:51.841948214Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:42:51.844890604Z","region_metadata_checked_at":"2026-09-30T00:42:51.844890604Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"69ed9c3164640f65a2f7286fe40e500d1dbccf4e1406858e01d642eb64146135","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052569519788032","occurred_at":"2026-09-29T21:50:17Z","failed_at":"e2e","signature_id":"4018967fd150f5152493725d6dbddb7a3c1f1594faf65bf4b39b51a0a8900bf9","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006fe4e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-q2dnllpw9nvn\" to finish creating: GET https://rp.j9788032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e0f2d148-6c09-4fa7-92eb-2682cee14684\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e0f2d148-6c09-4fa7-92eb-2682cee14684\",\n \"name\": \"e0f2d148-6c09-4fa7-92eb-2682cee14684\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:12:20.427830985Z\",\n \"endTime\": \"2026-09-29T23:31:23.466880978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-wjghl-dvxj8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-q2dnllpw9nvn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-wjghl-dvxj8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-wjghl-dvxj8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-q2dnllpw9nvn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-wjghl-dvxj8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006fe4e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-q2dnllpw9nvn\" to finish creating: GET https://rp.j9788032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e0f2d148-6c09-4fa7-92eb-2682cee14684\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e0f2d148-6c09-4fa7-92eb-2682cee14684\",\n \"name\": \"e0f2d148-6c09-4fa7-92eb-2682cee14684\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:12:20.427830985Z\",\n \"endTime\": \"2026-09-29T23:31:23.466880978Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-wjghl-dvxj8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-q2dnllpw9nvn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-wjghl-dvxj8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-wjghl-dvxj8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-q2dnllpw9nvn--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-wjghl-dvxj8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052566017544192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7183,"pr_state":"closed","pr_sha":"30a9a5e8593d1558577c9c639b734ef2df1555b8","final_merged_sha":"30a9a5e8593d1558577c9c639b734ef2df1555b8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T21:50:16Z","started_at":"2026-09-29T21:50:16Z","completed_at":"2026-09-30T00:18:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:32:16.223568778Z","timing_metadata_checked_at":"2026-09-30T00:32:16.223568778Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:32:16.226483169Z","region_metadata_checked_at":"2026-09-30T00:32:16.226483169Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052529241886720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T21:50:07Z","started_at":"2026-09-29T21:50:07Z","completed_at":"2026-09-30T00:18:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:32:15.592293088Z","timing_metadata_checked_at":"2026-09-30T00:32:15.592293088Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:32:15.596280576Z","region_metadata_checked_at":"2026-09-30T00:32:15.596280576Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052522010906624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7244,"pr_state":"closed","pr_sha":"cbddf39100b6fb0f90835d16a76eb4cb925cc7df","final_merged_sha":"cbddf39100b6fb0f90835d16a76eb4cb925cc7df","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T21:50:05Z","started_at":"2026-09-29T21:50:05Z","completed_at":"2026-09-30T00:12:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:21:59.899984431Z","timing_metadata_checked_at":"2026-09-30T00:21:59.899984431Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:21:59.903555821Z","region_metadata_checked_at":"2026-09-30T00:21:59.903555821Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"348453fda611d9bbac1d9e593cea09d35474c3a1890e86502849a94fbb810f56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052522010906624","occurred_at":"2026-09-29T21:50:05Z","failed_at":"e2e","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052482278264832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T21:49:56Z","started_at":"2026-09-29T21:49:56Z","completed_at":"2026-09-30T00:25:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:32:16.070522341Z","timing_metadata_checked_at":"2026-09-30T00:32:16.070522341Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:32:16.073232133Z","region_metadata_checked_at":"2026-09-30T00:32:16.073232133Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"1a21486b641eca52de28a7712e53dffd26d69c6531e9a97fe4487c71db630704","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052482278264832","occurred_at":"2026-09-29T21:49:56Z","failed_at":"alert","signature_id":"518ef70a52039b28f07c15b68bbea0314c0eae7e1ab6a7718953985817939a1d","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-30T00:09:36Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8264832-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8264832/providers/microsoft.monitor/accounts/services-j8264832\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-7bf4cc4544-w8cbp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j8264832-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-30T00:09:36Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j8264832-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8264832/providers/microsoft.monitor/accounts/services-j8264832\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-7bf4cc4544-w8cbp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j8264832-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052457632534528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7241,"pr_state":"closed","pr_sha":"ee4479e302079878c9a9758a921ca483cadac619","final_merged_sha":"ee4479e302079878c9a9758a921ca483cadac619","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T21:49:50Z","started_at":"2026-09-29T21:49:50Z","completed_at":"2026-09-30T00:26:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:42:52.293735006Z","timing_metadata_checked_at":"2026-09-30T00:42:52.293735006Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:42:52.300457085Z","region_metadata_checked_at":"2026-09-30T00:42:52.300457085Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052413286158336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T21:49:39Z","started_at":"2026-09-29T21:49:39Z","completed_at":"2026-09-30T00:16:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:32:15.783022011Z","timing_metadata_checked_at":"2026-09-30T00:32:15.783022011Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:32:15.786487201Z","region_metadata_checked_at":"2026-09-30T00:32:15.786487201Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7faacb95477f11c0d6c07adf91746b13608a6de44d6ff43d1dd91b25a4961c5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052413286158336","occurred_at":"2026-09-29T21:49:39Z","failed_at":"e2e","signature_id":"a2e6f33b9531e79917bad79473ad5885c1320f2309e0ab194f79b924ef0bae9f","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:249]: failed to create breakglass credentials for another user\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00095e460\u003e: \n failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-rsq2rx94fvrl/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-bcnzj/kubeconfig: timeout waiting for session to become ready (last status: {\"status\":\"Session is not ready\"})\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:249]: failed to create breakglass credentials for another user\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00095e460\u003e: \n failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-rsq2rx94fvrl/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-bcnzj/kubeconfig: timeout waiting for session to become ready (last status: {\"status\":\"Session is not ready\"})\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2923293c89e6d0b4f0db452002506cef1ee74dfd4af25725a8ed3b2648905b4","failure_pattern":"breakglass session kubeconfig not ready: timeout waiting for session to become ready","search_query":"failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-rsq2rx94fvrl/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-bcnzj/kubeconfig: timeout waiting for session to become ready (last status: {\"status\":\"Session is not ready\"})"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f2923293c89e6d0b4f0db452002506cef1ee74dfd4af25725a8ed3b2648905b4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052407825174528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T21:49:38Z","started_at":"2026-09-29T21:49:38Z","completed_at":"2026-09-30T00:35:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:42:52.0068978Z","timing_metadata_checked_at":"2026-09-30T00:42:52.0068978Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:42:52.010366589Z","region_metadata_checked_at":"2026-09-30T00:42:52.010366589Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105052394847997952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"7cc3bc8a1c5939f7125d9af7d41f712d1e116e4e","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-29T21:49:35Z","started_at":"2026-09-29T21:49:35Z","completed_at":"2026-09-30T00:20:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:32:15.938023542Z","timing_metadata_checked_at":"2026-09-30T00:32:15.938023542Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:32:15.941020333Z","region_metadata_checked_at":"2026-09-30T00:32:15.941020333Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105049386189852672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"closed","pr_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","final_merged_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T21:37:38Z","started_at":"2026-09-29T21:37:38Z","completed_at":"2026-09-29T23:58:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:11:37.195918039Z","timing_metadata_checked_at":"2026-09-30T00:11:37.195918039Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:11:37.199680427Z","region_metadata_checked_at":"2026-09-30T00:11:37.199680427Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ee822fb8e2723402a12d64e4d29a8074157ecd0950246eb59c3ed81a9f173b16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105049386189852672","occurred_at":"2026-09-29T21:37:38Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T22:45:32Z\n Ended: 2026-09-29T22:56:28Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9852672-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7219/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105047872234852352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7219,"pr_state":"open","pr_sha":"7b2a747f302543e6bfe2e701f5c304a8b4641ac5","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-29T21:31:37Z","started_at":"2026-09-29T21:31:37Z","completed_at":"2026-09-29T23:58:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:11:37.324707534Z","timing_metadata_checked_at":"2026-09-30T00:11:37.324707534Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:11:37.327371925Z","region_metadata_checked_at":"2026-09-30T00:11:37.327371925Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105039456577785856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7231,"pr_state":"open","pr_sha":"6b2f119abf63023e73f9cb51b51eeed1528473cf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T20:58:10Z","started_at":"2026-09-29T20:58:10Z","completed_at":"2026-09-29T23:04:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T23:19:37.180730708Z","timing_metadata_checked_at":"2026-09-29T23:19:37.180730708Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T23:19:37.186208492Z","region_metadata_checked_at":"2026-09-29T23:19:37.186208492Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"8c6fbe907a0c4cdf71c0b1daa2c3b39e5a296e30877667f3193b7677eb99fc77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105039456577785856","occurred_at":"2026-09-29T20:58:10Z","failed_at":"e2e","signature_id":"980181d054e183e3e9c68c84d9a663132a5bf02948b58946b7ea84d0771608b3","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:151]: failed to verify cluster health or IngressController scope for cluster \"pi-4-21\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001589cb0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:151]: failed to verify cluster health or IngressController scope for cluster \"pi-4-21\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001589cb0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}},{"row_id":"cc62e9cb0984b28499f134a8a685e0ad809521be713b1b9a752ab1e055042fe0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105039456577785856","occurred_at":"2026-09-29T20:58:10Z","failed_at":"e2e","signature_id":"f81475c92059926a1e5118cae23540a3070f6b365e99f3d9f58c8d456b102cce","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:151]: failed to verify cluster health or IngressController scope for cluster \"pi-4-20\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00122eff0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:151]: failed to verify cluster health or IngressController scope for cluster \"pi-4-20\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00122eff0\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105037155771355136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"0bdd36fc9bed8d937676c8592b848cd9cbcde70d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T20:48:41Z","started_at":"2026-09-29T20:48:41Z","completed_at":"2026-09-29T22:59:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T23:08:48.087462457Z","timing_metadata_checked_at":"2026-09-29T23:08:48.087462457Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T23:08:48.091584945Z","region_metadata_checked_at":"2026-09-29T23:08:48.091584945Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"563111598b7aef449479515320358a5e2cd5ab0012f8bf12fa57d9fb80d28bbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105037155771355136","occurred_at":"2026-09-29T20:48:41Z","failed_at":"e2e","signature_id":"fc0b9e52dc41a1cfd3d393b841e137e164b826bc9a9e2d4c2bddbc73f64e19ef","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:740]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001071820\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-pxnddw\" in resourcegroup=\"rg-np-dg-minor-pxnddw-dsdsdnq4v6ch\" to finish creating: GET https://rp.j1355136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bedd488d-acf4-4a7b-9e30-cb9921b62867\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bedd488d-acf4-4a7b-9e30-cb9921b62867\",\n \"name\": \"bedd488d-acf4-4a7b-9e30-cb9921b62867\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T22:05:15.883513695Z\",\n \"endTime\": \"2026-09-29T22:24:16.137353481Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q1b0s9q1x0x2p8w-npdg-422-420-t77mj-czdm4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-pxnddw-dsdsdnq4v6ch-np-dgm-pxnddw--managed/providers/Microsoft.Compute/virtualMachines/q1b0s9q1x0x2p8w-npdg-422-420-t77mj-czdm4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q1b0s9q1x0x2p8w-npdg-422-420-t77mj-czdm4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-pxnddw-dsdsdnq4v6ch-np-dgm-pxnddw--managed/providers/Microsoft.Compute/virtualMachines/q1b0s9q1x0x2p8w-npdg-422-420-t77mj-czdm4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:740]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001071820\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-pxnddw\" in resourcegroup=\"rg-np-dg-minor-pxnddw-dsdsdnq4v6ch\" to finish creating: GET https://rp.j1355136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bedd488d-acf4-4a7b-9e30-cb9921b62867\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bedd488d-acf4-4a7b-9e30-cb9921b62867\",\n \"name\": \"bedd488d-acf4-4a7b-9e30-cb9921b62867\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T22:05:15.883513695Z\",\n \"endTime\": \"2026-09-29T22:24:16.137353481Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q1b0s9q1x0x2p8w-npdg-422-420-t77mj-czdm4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-pxnddw-dsdsdnq4v6ch-np-dgm-pxnddw--managed/providers/Microsoft.Compute/virtualMachines/q1b0s9q1x0x2p8w-npdg-422-420-t77mj-czdm4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q1b0s9q1x0x2p8w-npdg-422-420-t77mj-czdm4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-pxnddw-dsdsdnq4v6ch-np-dgm-pxnddw--managed/providers/Microsoft.Compute/virtualMachines/q1b0s9q1x0x2p8w-npdg-422-420-t77mj-czdm4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105034371663663104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"966ce9ad077e9c54157a400740c52c1ae67fc234","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-29T20:37:58Z","started_at":"2026-09-29T20:37:58Z","completed_at":"2026-09-29T22:53:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T23:08:48.271144901Z","timing_metadata_checked_at":"2026-09-29T23:08:48.271144901Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T23:08:48.279735175Z","region_metadata_checked_at":"2026-09-29T23:08:48.279735175Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7251/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105032680448987136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7251,"pr_state":"closed","pr_sha":"766e0ef25d26bc8fad353f644663787115a38b6f","final_merged_sha":"766e0ef25d26bc8fad353f644663787115a38b6f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T20:31:15Z","started_at":"2026-09-29T20:31:15Z","completed_at":"2026-09-29T22:44:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T22:58:45.473299394Z","timing_metadata_checked_at":"2026-09-29T22:58:45.473299394Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T22:58:45.476028587Z","region_metadata_checked_at":"2026-09-29T22:58:45.476028587Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"83463c1478892223a32a9d8cee8f7ed10e15d9f2234658c8ee1958a4efeccef7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7251/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105032680448987136","occurred_at":"2026-09-29T20:31:15Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T21:30:31Z\n Ended: 2026-09-29T21:40:23Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8987136-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105029879664742400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7250,"pr_state":"open","pr_sha":"96ca57cb7d5eab9eb9db1b06eaec792e6f8685b8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T20:15:30Z","started_at":"2026-09-29T20:15:30Z","completed_at":"2026-09-29T22:38:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T22:58:45.34568336Z","timing_metadata_checked_at":"2026-09-29T22:58:45.34568336Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T22:58:45.34932445Z","region_metadata_checked_at":"2026-09-29T22:58:45.34932445Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"86d46c701d875d0110e6eee33585c38d60a301cf0aced3d7f483f5fa5b26fbca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7250/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105029879664742400","occurred_at":"2026-09-29T20:15:30Z","failed_at":"e2e","signature_id":"002b290be3c71f7a1bf61c63b79aa5aececd77559baecd239044cd92c478902e","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b835a0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-d25xmd5lmjzd\" to finish creating: GET https://rp.j4742400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a260492b-096c-44ec-a66c-3ad97273eaa0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a260492b-096c-44ec-a66c-3ad97273eaa0\",\n \"name\": \"a260492b-096c-44ec-a66c-3ad97273eaa0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T21:43:34.965858019Z\",\n \"endTime\": \"2026-09-29T22:02:41.653989842Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-7hh2h-jn4bt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-d25xmd5lmjzd--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-7hh2h-jn4bt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-7hh2h-jn4bt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-d25xmd5lmjzd--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-7hh2h-jn4bt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b835a0\u003e: \n failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-d25xmd5lmjzd\" to finish creating: GET https://rp.j4742400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a260492b-096c-44ec-a66c-3ad97273eaa0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a260492b-096c-44ec-a66c-3ad97273eaa0\",\n \"name\": \"a260492b-096c-44ec-a66c-3ad97273eaa0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T21:43:34.965858019Z\",\n \"endTime\": \"2026-09-29T22:02:41.653989842Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-7hh2h-jn4bt: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-d25xmd5lmjzd--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-7hh2h-jn4bt to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-7hh2h-jn4bt: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-d25xmd5lmjzd--managed/providers/Microsoft.Compute/virtualMachines/oidc-wi-cluster-oidc-wi-np-7hh2h-jn4bt to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105026534115905536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7104,"pr_state":"closed","pr_sha":"a3c6e7c2bdf35fc4c5ed664a500c0ae1ae8af1be","final_merged_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-29T20:03:00Z","started_at":"2026-09-29T20:03:00Z","completed_at":"2026-09-29T22:34:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T22:48:17.258610968Z","timing_metadata_checked_at":"2026-09-29T22:48:17.258610968Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T22:48:17.26191786Z","region_metadata_checked_at":"2026-09-29T22:48:17.26191786Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105029025452789760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7231,"pr_state":"open","pr_sha":"6b2f119abf63023e73f9cb51b51eeed1528473cf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T20:00:48Z","started_at":"2026-09-29T20:00:48Z","completed_at":"2026-09-29T20:54:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T21:03:40.938852264Z","timing_metadata_checked_at":"2026-09-29T21:03:40.938852264Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T21:03:40.941724355Z","region_metadata_checked_at":"2026-09-29T21:03:40.941724355Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bb380c9d219362697295710664cc0bf32f7cbda970b7503c13e0fa3cd3e6d021","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105029025452789760","occurred_at":"2026-09-29T20:00:48Z","failed_at":"provision","signature_id":"6b8d9984b55592851b0841c5e8c4a0f18f8f61520e7df4a51c7e6afc9a707725","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/kube-events","test_suite":"templatize-pipeline","failure_text":"time=2026-09-29T20:43:19.784Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events description=\"Step kube-events\\n Kind: Helm\\n\"\ntime=2026-09-29T20:43:20.131Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events values=\"map[enabled:true image:map[digest:sha256:54b14654077ec8c910ff0eeeda191bb45421c6496b2211751fbfd13b38779a46 registry:kubernetesshared.azurecr.io repository:shared/kube-events] resources:map[limits:map[memory:unlimited] requests:map[cpu:200m memory:512Mi]]]\"\ntime=2026-09-29T20:43:20.131Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-09-29T20:43:20.307Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events namespace=monitoring\ntime=2026-09-29T20:43:20.477Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events dryRun=false\ntime=2026-09-29T20:43:20.477Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-09-29T20:43:20.513Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events\ntime=2026-09-29T20:48:23.298Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"\ntime=2026-09-29T20:48:23.298Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=kube-events err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a","failure_pattern":"failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1 context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/monitoring/kube-events not ready. status: InProgress, message: Available: 0/1\\ncontext deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dac8a1b25a3188542fe843437616ad526549332051ce7885378b1b60aa94182a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105026278703763456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7238,"pr_state":"closed","pr_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","final_merged_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T19:19:00Z","started_at":"2026-09-29T19:19:00Z","completed_at":"2026-09-29T22:30:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T22:48:17.278797119Z","timing_metadata_checked_at":"2026-09-29T22:48:17.278797119Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T22:48:17.281218714Z","region_metadata_checked_at":"2026-09-29T22:48:17.281218714Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b4146c77d8781d544aceaf379d8babc4da63195a4c57b5d1a79278c83a83860e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105026278703763456","occurred_at":"2026-09-29T19:19:00Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T21:12:57Z\n Ended: 2026-09-29T21:27:58Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3763456-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105024277664239616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7242,"pr_state":"closed","pr_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","final_merged_sha":"136c196e662abb71c58ecca8b6794334296a7ffd","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T19:18:47Z","started_at":"2026-09-29T19:18:47Z","completed_at":"2026-09-29T22:28:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T22:48:17.300009368Z","timing_metadata_checked_at":"2026-09-29T22:48:17.300009368Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T22:48:17.302387962Z","region_metadata_checked_at":"2026-09-29T22:48:17.302387962Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"22e364d864fb7f058c48790c2e705daa6f57b03bab0efbe561c8021141d665a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7242/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105024277664239616","occurred_at":"2026-09-29T19:18:47Z","failed_at":"e2e","signature_id":"0f4293ef89ffa047949ef502c6da1fc06a44e247ae7165e198a6f41e82957ed1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002a9c120\u003e: \n resource group \"feature-aggregation-58s9mcwldsqm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-58s9mcwldsqm\" to finish deleting: GET https://rp.j4239616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5803e91-8e02-4c30-b145-ec8ce0a8c23b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5803e91-8e02-4c30-b145-ec8ce0a8c23b\",\n \"name\": \"c5803e91-8e02-4c30-b145-ec8ce0a8c23b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T21:25:15.122990062Z\",\n \"endTime\": \"2026-09-29T21:49:22.352588907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4tf85mf6cm5no630bapiv0pq2r6pgq still exists (deletion dispatched at 2026-09-29T21:25:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002a9c120\u003e: \n resource group \"feature-aggregation-58s9mcwldsqm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-58s9mcwldsqm\" to finish deleting: GET https://rp.j4239616.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5803e91-8e02-4c30-b145-ec8ce0a8c23b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c5803e91-8e02-4c30-b145-ec8ce0a8c23b\",\n \"name\": \"c5803e91-8e02-4c30-b145-ec8ce0a8c23b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T21:25:15.122990062Z\",\n \"endTime\": \"2026-09-29T21:49:22.352588907Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4tf85mf6cm5no630bapiv0pq2r6pgq still exists (deletion dispatched at 2026-09-29T21:25:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105012837859463168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"8470e4a8152fec1ad8e98ecdb32250dd598096a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T18:44:38Z","started_at":"2026-09-29T18:44:38Z","completed_at":"2026-09-29T21:26:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T21:45:34.409252602Z","timing_metadata_checked_at":"2026-09-29T21:45:34.409252602Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T21:45:34.412544191Z","region_metadata_checked_at":"2026-09-29T21:45:34.412544191Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9038cdec8007571a19c77dbd90ed90b936a4fdd4e70a5acdf42d94a39d47a73a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105012837859463168","occurred_at":"2026-09-29T18:44:38Z","failed_at":"e2e","signature_id":"0112ee342011d5c23546c79807c716164af38d0c4d3fb5749f3293d26410c442","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000c8f800\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000c8f800\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"bed0cbb3c3bdaf51ae87b83bc701ce4944415fe6d322035f418dda60d197f8e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105012837859463168","occurred_at":"2026-09-29T18:44:38Z","failed_at":"e2e","signature_id":"2dfd5b2fbf1861319ad8698d5e0ecdc3fef355379167aee3783a7184e0aeef07","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003710200\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-mvqbmn55cv5b\" to finish creating: GET https://rp.j9463168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/adec164f-ea16-461c-9762-733a451ff4ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/adec164f-ea16-461c-9762-733a451ff4ec\",\n \"name\": \"adec164f-ea16-461c-9762-733a451ff4ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T20:11:35.519536155Z\",\n \"endTime\": \"2026-09-29T20:30:41.781416805Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n0c3u0l0p2x4r6y-np-deletable-4d2w7-wx26q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-mvqbmn55cv5b--managed/providers/Microsoft.Compute/virtualMachines/n0c3u0l0p2x4r6y-np-deletable-4d2w7-wx26q to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n0c3u0l0p2x4r6y-np-deletable-4d2w7-wx26q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-mvqbmn55cv5b--managed/providers/Microsoft.Compute/virtualMachines/n0c3u0l0p2x4r6y-np-deletable-4d2w7-wx26q to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003710200\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-mvqbmn55cv5b\" to finish creating: GET https://rp.j9463168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/adec164f-ea16-461c-9762-733a451ff4ec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/adec164f-ea16-461c-9762-733a451ff4ec\",\n \"name\": \"adec164f-ea16-461c-9762-733a451ff4ec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T20:11:35.519536155Z\",\n \"endTime\": \"2026-09-29T20:30:41.781416805Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n0c3u0l0p2x4r6y-np-deletable-4d2w7-wx26q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-mvqbmn55cv5b--managed/providers/Microsoft.Compute/virtualMachines/n0c3u0l0p2x4r6y-np-deletable-4d2w7-wx26q to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n0c3u0l0p2x4r6y-np-deletable-4d2w7-wx26q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-mvqbmn55cv5b--managed/providers/Microsoft.Compute/virtualMachines/n0c3u0l0p2x4r6y-np-deletable-4d2w7-wx26q to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-mvqbmn55cv5b\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7245/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105010614010122240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7245,"pr_state":"closed","pr_sha":"f123239b74b59b29eef0693165ee8bff8c1e6522","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-29T18:39:40Z","started_at":"2026-09-29T18:39:40Z","completed_at":"2026-09-29T21:17:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T21:24:33.700936136Z","timing_metadata_checked_at":"2026-09-29T21:24:33.700936136Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T21:24:33.705648523Z","region_metadata_checked_at":"2026-09-29T21:24:33.705648523Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105006438400135168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"7cc3bc8a1c5939f7125d9af7d41f712d1e116e4e","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T18:25:20Z","started_at":"2026-09-29T18:25:20Z","completed_at":"2026-09-29T21:03:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T21:24:33.549827463Z","timing_metadata_checked_at":"2026-09-29T21:24:33.549827463Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T21:24:33.553106054Z","region_metadata_checked_at":"2026-09-29T21:24:33.553106054Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"6c514363b1a9e8a00c48bb92869376545df2b2e44562c31841e3f8171f8b0931","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105006438400135168","occurred_at":"2026-09-29T18:25:20Z","failed_at":"e2e","signature_id":"ffcbb348f85ac373b67501a40e7b6ae6c3f45dfcc35bf054eddc68c37318fe83","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001542840\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001542840\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"7d563062e82c0df3b5bd3d95d34403888177abe16ad8911604f14eccbc62a5ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105006438400135168","occurred_at":"2026-09-29T18:25:20Z","failed_at":"e2e","signature_id":"bf1f7178169b541464d692cd921bd634d6b51328b81b900fa0a4c18cde694577","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-b9pvc2cjcshp/cluster-candidate-4-23-nm2hv7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea4780\u003e: \n failed to create HCP cluster cluster-candidate-4-23-nm2hv7: failed waiting for cluster=\"cluster-candidate-4-23-nm2hv7\" in resourcegroup=\"rg-candidate-4-23-b9pvc2cjcshp\" to finish creating: GET https://rp.j0135168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7bbfe34a-1f3c-4ee3-924d-466a04a1edb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7bbfe34a-1f3c-4ee3-924d-466a04a1edb8\",\n \"name\": \"7bbfe34a-1f3c-4ee3-924d-466a04a1edb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T19:33:49.148595696Z\",\n \"endTime\": \"2026-09-29T20:02:49.618696201Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-b9pvc2cjcshp/cluster-candidate-4-23-nm2hv7 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea4780\u003e: \n failed to create HCP cluster cluster-candidate-4-23-nm2hv7: failed waiting for cluster=\"cluster-candidate-4-23-nm2hv7\" in resourcegroup=\"rg-candidate-4-23-b9pvc2cjcshp\" to finish creating: GET https://rp.j0135168.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7bbfe34a-1f3c-4ee3-924d-466a04a1edb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7bbfe34a-1f3c-4ee3-924d-466a04a1edb8\",\n \"name\": \"7bbfe34a-1f3c-4ee3-924d-466a04a1edb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T19:33:49.148595696Z\",\n \"endTime\": \"2026-09-29T20:02:49.618696201Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dd040b1e9c70e1ca44d5ea27f3b632dc652dd9f6dab2e8974f6ee7f9994b0f42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105006438400135168","occurred_at":"2026-09-29T18:25:20Z","failed_at":"alert","signature_id":"2baffeea20bd676008c8b5d31bd8c88b616a29eb6a5757e53af3aeca8a113911","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T19:56:54Z\n Ended: 2026-09-29T20:20:53Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0135168-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.15:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0135168/providers/microsoft.monitor/accounts/hcps-j0135168\", namespace=\"ocm-arohcpci01-2t4s97ialb0v6jm7ucotiecf3ule87m0-w2j1q7d0a6j3n6k\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-cnk2l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4s97ialb0v6jm7ucotiecf3ule87m0-w2j1q7d0a6j3n6k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T19:56:54Z\n Ended: 2026-09-29T20:24:52Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0135168-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.15:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0135168/providers/microsoft.monitor/accounts/hcps-j0135168\", namespace=\"ocm-arohcpci01-2t4s97ialb0v6jm7ucotiecf3ule87m0-w2j1q7d0a6j3n6k\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-cnk2l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4s97ialb0v6jm7ucotiecf3ule87m0-w2j1q7d0a6j3n6k/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"f33098a44598c167967b7fb0904a99e32ebc29bb660c1f1dc9afeb52fdbebde4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105006438400135168","occurred_at":"2026-09-29T18:25:20Z","failed_at":"alert","signature_id":"011948abe2f463f2b07cf5232a16746b84e5d9d9d536e02a5cd1423247a16032","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T20:10:50Z\n Ended: 2026-09-29T20:26:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0135168-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.15:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0135168/providers/microsoft.monitor/accounts/hcps-j0135168\", namespace=\"ocm-arohcpci01-2t4s97ialb0v6jm7ucotiecf3ule87m0-w2j1q7d0a6j3n6k\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-cnk2l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4s97ialb0v6jm7ucotiecf3ule87m0-w2j1q7d0a6j3n6k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105000598473543680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"0bdd36fc9bed8d937676c8592b848cd9cbcde70d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T18:00:13Z","started_at":"2026-09-29T18:00:13Z","completed_at":"2026-09-29T20:40:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T20:43:09.368018694Z","timing_metadata_checked_at":"2026-09-29T20:43:09.368018694Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T20:43:09.370489886Z","region_metadata_checked_at":"2026-09-29T20:43:09.370489886Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"7dc5a11995dd1e248ff8789435e725c9262dd6965a6be799e781628c63af370b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105000598473543680","occurred_at":"2026-09-29T18:00:13Z","failed_at":"e2e","signature_id":"c5550fc47ca461f8d74ae7e4533455a0892c5064108b09a047f57bccccb056fd","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00112a620\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-zg4mlcxxqtkw\" to finish creating: GET https://rp.j3543680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a237115-bd4b-4faf-99f3-a126cf693a5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a237115-bd4b-4faf-99f3-a126cf693a5b\",\n \"name\": \"9a237115-bd4b-4faf-99f3-a126cf693a5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T19:15:41.667113509Z\",\n \"endTime\": \"2026-09-29T19:44:46.805281429Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00112a620\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-zg4mlcxxqtkw\" to finish creating: GET https://rp.j3543680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a237115-bd4b-4faf-99f3-a126cf693a5b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9a237115-bd4b-4faf-99f3-a126cf693a5b\",\n \"name\": \"9a237115-bd4b-4faf-99f3-a126cf693a5b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T19:15:41.667113509Z\",\n \"endTime\": \"2026-09-29T19:44:46.805281429Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"89af520752f2f11e3b94e8cd6da98f95674c31f632736b747454a54fa97841f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105000598473543680","occurred_at":"2026-09-29T18:00:13Z","failed_at":"alert","signature_id":"668e41f6097bef845ab586e3a8a88ee93a1933e53b3b3588deccbca7faf09224","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T19:35:51Z\n Ended: 2026-09-29T20:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3543680-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3543680/providers/microsoft.monitor/accounts/hcps-j3543680\", namespace=\"ocm-arohcpci01-2t4s0nlb3kmtdnv66shggj311g6csrra-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-c4fhh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4s0nlb3kmtdnv66shggj311g6csrra-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"92e55484c931b0797d96c4f24b71f96e2a31d012debb9dd16c7df0a06e08057d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105000598473543680","occurred_at":"2026-09-29T18:00:13Z","failed_at":"alert","signature_id":"50517fc7fd024fc7e879e7001ae0748846509b3833e9c936fa72b45225cf416b","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T19:37:53Z\n Ended: 2026-09-29T20:07:50Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3543680-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3543680/providers/microsoft.monitor/accounts/hcps-j3543680\", namespace=\"ocm-arohcpci01-2t4s0nlb3kmtdnv66shggj311g6csrra-basic-cluster\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d56b7e50-ca7c-400c-bbda-078a761f4494\"\n Description: Pod ocm-arohcpci01-2t4s0nlb3kmtdnv66shggj311g6csrra-basic-cluster/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-29T19:37:53Z\n Ended: 2026-09-29T20:06:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3543680-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3543680/providers/microsoft.monitor/accounts/hcps-j3543680\", namespace=\"ocm-arohcpci01-2t4s0nlb3kmtdnv66shggj311g6csrra-basic-cluster\", pod=\"etcd-2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d56b7e50-ca7c-400c-bbda-078a761f4494\"\n Description: Pod ocm-arohcpci01-2t4s0nlb3kmtdnv66shggj311g6csrra-basic-cluster/etcd-2 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"a82826f3cdc046d657417cb3fc76bb79e6fd7fd361ac1fff0d156c0f45f511c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2105000598473543680","occurred_at":"2026-09-29T18:00:13Z","failed_at":"e2e","signature_id":"07e31f9b332fdcf32228351bebed395b46cc4e47503ddef233b8a947306f8350","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001065710\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001065710\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104997947618889728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7124,"pr_state":"closed","pr_sha":"113526cb8f77445e2427551f69427f9bef948532","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T17:48:54Z","started_at":"2026-09-29T17:48:54Z","completed_at":"2026-09-29T20:33:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T20:43:09.583682574Z","timing_metadata_checked_at":"2026-09-29T20:43:09.583682574Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T20:43:09.586373465Z","region_metadata_checked_at":"2026-09-29T20:43:09.586373465Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"63739913d3d2b0ba5fdbe7429ee6696258dc524525cdfe28ddf9cd1dd73f881a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104997947618889728","occurred_at":"2026-09-29T17:48:54Z","failed_at":"e2e","signature_id":"97c003c7505eb6d1fea87917d772800ab3a4391c3491f8d46624d6aed1c89673","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013f42e8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013f42e8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"daba989f5c07b9d5570db246f74c8a1d9be2e34f6e770304fbe434c175a6079a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104997947618889728","occurred_at":"2026-09-29T17:48:54Z","failed_at":"e2e","signature_id":"875c9de593bf1b0f10867e9b319fa7c87ab182e48aba7abf0df135714d7ebaf9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pvrwz4\" for cluster \"cluster-candidate-4-22-pvrwz4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009d8660\u003e: \n failed to create NodePool np-pvrwz4: failed waiting for nodepool=\"np-pvrwz4\" for cluster \"cluster-candidate-4-22-pvrwz4\" in resourcegroup=\"rg-candidate-4-22-d42kl4z7ng82\" to finish creating: GET https://rp.j8889728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0776c15e-ebd8-4b09-8f3b-40299621b51b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0776c15e-ebd8-4b09-8f3b-40299621b51b\",\n \"name\": \"0776c15e-ebd8-4b09-8f3b-40299621b51b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T19:13:29.357971137Z\",\n \"endTime\": \"2026-09-29T19:32:31.78446481Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-4zzq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d42kl4z7ng82-candidate-pvrwz4--managed/providers/Microsoft.Compute/virtualMachines/m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-4zzq4 to exist\\nMachine m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-5zgq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d42kl4z7ng82-candidate-pvrwz4--managed/providers/Microsoft.Compute/virtualMachines/m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-5zgq4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-4zzq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d42kl4z7ng82-candidate-pvrwz4--managed/providers/Microsoft.Compute/virtualMachines/m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-4zzq4 to exist\\nMachine m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-5zgq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d42kl4z7ng82-candidate-pvrwz4--managed/providers/Microsoft.Compute/virtualMachines/m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-5zgq4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pvrwz4\" for cluster \"cluster-candidate-4-22-pvrwz4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009d8660\u003e: \n failed to create NodePool np-pvrwz4: failed waiting for nodepool=\"np-pvrwz4\" for cluster \"cluster-candidate-4-22-pvrwz4\" in resourcegroup=\"rg-candidate-4-22-d42kl4z7ng82\" to finish creating: GET https://rp.j8889728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0776c15e-ebd8-4b09-8f3b-40299621b51b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0776c15e-ebd8-4b09-8f3b-40299621b51b\",\n \"name\": \"0776c15e-ebd8-4b09-8f3b-40299621b51b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T19:13:29.357971137Z\",\n \"endTime\": \"2026-09-29T19:32:31.78446481Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-4zzq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d42kl4z7ng82-candidate-pvrwz4--managed/providers/Microsoft.Compute/virtualMachines/m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-4zzq4 to exist\\nMachine m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-5zgq4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d42kl4z7ng82-candidate-pvrwz4--managed/providers/Microsoft.Compute/virtualMachines/m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-5zgq4 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-4zzq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d42kl4z7ng82-candidate-pvrwz4--managed/providers/Microsoft.Compute/virtualMachines/m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-4zzq4 to exist\\nMachine m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-5zgq4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d42kl4z7ng82-candidate-pvrwz4--managed/providers/Microsoft.Compute/virtualMachines/m4d9b1g6f0p9j7r-np-pvrwz4-bfbpc-5zgq4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104995028081315840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"5d8019a38c7d14b05dd0bd25eecdfdb5a712b411","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T17:29:41Z","started_at":"2026-09-29T17:29:41Z","completed_at":"2026-09-29T20:34:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T20:43:09.007969697Z","timing_metadata_checked_at":"2026-09-29T20:43:09.007969697Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T20:43:09.010859688Z","region_metadata_checked_at":"2026-09-29T20:43:09.010859688Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e5da79ce214024da039170763cfdf59d910a2eafcb0cc5306e69dd0d0ba994c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104995028081315840","occurred_at":"2026-09-29T17:29:41Z","failed_at":"e2e","signature_id":"f62c50cf70826e4bfb06d0aff031079f947dcaa536f6c525d86fdbdc131332ad","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001682360\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001682360\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104978275775287296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7200,"pr_state":"open","pr_sha":"228b9f7e3ef7f08eca81b6ab0ffa197893c7a7ff","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T16:55:03Z","started_at":"2026-09-29T16:55:03Z","completed_at":"2026-09-29T19:20:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T19:29:09.543016392Z","timing_metadata_checked_at":"2026-09-29T19:29:09.543016392Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T19:29:09.548410777Z","region_metadata_checked_at":"2026-09-29T19:29:09.548410777Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d287b8a9b1ddbd9d8af6ab26fb78c75190692d3ea2d02c7a78dc6e3a83decb5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104978275775287296","occurred_at":"2026-09-29T16:55:03Z","failed_at":"e2e","signature_id":"43ac23f8a40d803d5985dc9c930ebe956846e67cc5f0cf24d6e9f86705fe5a61","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.005s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000d2b7b8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.005s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000d2b7b8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977540106948608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7241,"pr_state":"closed","pr_sha":"ee4479e302079878c9a9758a921ca483cadac619","final_merged_sha":"ee4479e302079878c9a9758a921ca483cadac619","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T16:52:08Z","started_at":"2026-09-29T16:52:08Z","completed_at":"2026-09-29T19:31:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T19:40:06.527021995Z","timing_metadata_checked_at":"2026-09-29T19:40:06.527021995Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T19:40:06.530078689Z","region_metadata_checked_at":"2026-09-29T19:40:06.530078689Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"bd1babc1d4e0c663943d166f0ae528220c7aa0b6760dc84c6083ec498b6ec88c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977540106948608","occurred_at":"2026-09-29T16:52:08Z","failed_at":"e2e","signature_id":"c42e537ffd6d4eec5cd4d2c4e6dc9732ef01ce726dbb36309bb75486855b7442","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000a18c00\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000a18c00\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"d728520ae459e6587ce74cea7ea2117e57b128809e74e70073f25dd0996eba22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977540106948608","occurred_at":"2026-09-29T16:52:08Z","failed_at":"e2e","signature_id":"42b8a91f761a09a88ae01f9b8a9c7279d90c5604a9bde9eb96a04a1c749bb721","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000d0e4a0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:47677-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.002s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000d0e4a0\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:47677-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977492128305152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"e7500a3658c1692b6c32705cb7edf403b704aae4","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T16:51:57Z","started_at":"2026-09-29T16:51:57Z","completed_at":"2026-09-29T19:22:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T19:29:09.395502705Z","timing_metadata_checked_at":"2026-09-29T19:29:09.395502705Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T19:29:09.404671579Z","region_metadata_checked_at":"2026-09-29T19:29:09.404671579Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c9117930f8f291ce4024602607b451f5497c2da8823da0bcc54ad324ece9542b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977492128305152","occurred_at":"2026-09-29T16:51:57Z","failed_at":"e2e","signature_id":"3e46eaebf1c7e155f0af0a04d0d9fd5f9528b92fce51326d6aa350d33708f536","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0012fe180\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0012fe180\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977428710428672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T16:51:34Z","started_at":"2026-09-29T16:51:34Z","completed_at":"2026-09-29T19:29:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T19:40:06.202336308Z","timing_metadata_checked_at":"2026-09-29T19:40:06.202336308Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T19:40:06.205508301Z","region_metadata_checked_at":"2026-09-29T19:40:06.205508301Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"69fa1e005a81847934b92fe9a48c1943c9f4dd06218cd939b79fc2f982672a79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977428710428672","occurred_at":"2026-09-29T16:51:34Z","failed_at":"e2e","signature_id":"573950b7da00dec43bb410f800f47a6634208a43168239980f1b47fc7731f307","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:249]: failed to create breakglass credentials for another user\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019f000\u003e: \n failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-xcdsf8s6jfv2/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-94w82/kubeconfig: timeout waiting for session to become ready (last status: {\"status\":\"Session is not ready\"})\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:249]: failed to create breakglass credentials for another user\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00019f000\u003e: \n failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-xcdsf8s6jfv2/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-94w82/kubeconfig: timeout waiting for session to become ready (last status: {\"status\":\"Session is not ready\"})\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2923293c89e6d0b4f0db452002506cef1ee74dfd4af25725a8ed3b2648905b4","failure_pattern":"breakglass session kubeconfig not ready: timeout waiting for session to become ready","search_query":"failed to get ready session kubeconfig from /admin/v1/hcp/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/admin-api-breakglass-rsq2rx94fvrl/providers/microsoft.redhatopenshift/hcpopenshiftclusters/sre-hcp-cluster/breakglass/breakglass-bcnzj/kubeconfig: timeout waiting for session to become ready (last status: {\"status\":\"Session is not ready\"})"}},{"row_id":"7d835206abd1615b0395ad5c0573c436a59518374d834934b8b269b7c9bfc38f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977428710428672","occurred_at":"2026-09-29T16:51:34Z","failed_at":"e2e","signature_id":"8bf03abc3d2f87dcb7a75b59f8408b02d7ba3e53c3c6f90bf726e885bc3262fa","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000d609c0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000d609c0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","f2923293c89e6d0b4f0db452002506cef1ee74dfd4af25725a8ed3b2648905b4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977337324933120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7238,"pr_state":"closed","pr_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","final_merged_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T16:51:20Z","started_at":"2026-09-29T16:51:20Z","completed_at":"2026-09-29T19:10:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T19:18:37.201780124Z","timing_metadata_checked_at":"2026-09-29T19:18:37.201780124Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T19:18:37.204735414Z","region_metadata_checked_at":"2026-09-29T19:18:37.204735414Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"49befe0cdf05001cc86436c0bf985c83b5c725ac301f54e5de7c866fefc4fdf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104977337324933120","occurred_at":"2026-09-29T16:51:20Z","failed_at":"e2e","signature_id":"929f32feaec99ef600e64b43c8f15f46f4e06ba9e9caeab61d1d519adcb74159","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0012c9560\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0012c9560\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7236/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104976959313285120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7236,"pr_state":"closed","pr_sha":"d5cf9a520986ae23608f3ac67830e681c126c99c","final_merged_sha":"d5cf9a520986ae23608f3ac67830e681c126c99c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T16:49:50Z","started_at":"2026-09-29T16:49:50Z","completed_at":"2026-09-29T19:05:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T19:08:27.703477908Z","timing_metadata_checked_at":"2026-09-29T19:08:27.703477908Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T19:08:27.706780898Z","region_metadata_checked_at":"2026-09-29T19:08:27.706780898Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7246/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104971730622615552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7246,"pr_state":"closed","pr_sha":"26be4d462d878ae28865cbe5d86888fee258acfb","final_merged_sha":"26be4d462d878ae28865cbe5d86888fee258acfb","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T16:29:03Z","started_at":"2026-09-29T16:29:03Z","completed_at":"2026-09-29T18:39:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T18:58:07.397418828Z","timing_metadata_checked_at":"2026-09-29T18:58:07.397418828Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T18:58:07.407569395Z","region_metadata_checked_at":"2026-09-29T18:58:07.407569395Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"49ec3f1aaeaa8363c7cf4d1614113811b94a3547f0bd39b78b4bf6f9fe4c16ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7246/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104971730622615552","occurred_at":"2026-09-29T16:29:03Z","failed_at":"alert","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T17:28:34Z\n Ended: 2026-09-29T17:45:34Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2615552-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T17:35:35Z\n Ended: 2026-09-29T17:48:39Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2615552-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"9ee67b6b984802701124359f9472683436ef83e3d9e06291a463d5a7fc30c6e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7246/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104971730622615552","occurred_at":"2026-09-29T16:29:03Z","failed_at":"e2e","signature_id":"6a74ddb0d4242e1220e9ff7d24e971492c9841cf2cf07ad776ee04d824ac26db","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:178]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000a45800\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:178]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000a45800\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"b31d1beb59bff387590cbcbf9e6a52530a67f89b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T16:20:47Z","started_at":"2026-09-29T16:20:47Z","completed_at":"2026-09-29T18:58:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T19:08:27.530593095Z","timing_metadata_checked_at":"2026-09-29T19:08:27.530593095Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T19:08:27.533787286Z","region_metadata_checked_at":"2026-09-29T19:08:27.533787286Z"},"failed_at":["alert","e2e"],"failed_test_count":14,"occurrences":[{"row_id":"05866dcbd423d2c31efed91f8c0dc9218303111de1a3d6bdf15a852c6bab0b76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"8dadd23d7d92215c005555854307fa496e16f2ecc2d9f90b4ae5c21d9ef80068","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c2a078\u003e: \n resource group \"private-keyvault-mgxpmjvchvxg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mgxpmjvchvxg\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e9da9cc-6ed2-496f-a887-421a5555582f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e9da9cc-6ed2-496f-a887-421a5555582f\",\n \"name\": \"3e9da9cc-6ed2-496f-a887-421a5555582f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:41:56.828729728Z\",\n \"endTime\": \"2026-09-29T18:06:02.29902555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q9dplvqkvipc2q8n5pplrfqn14c16 still exists (deletion dispatched at 2026-09-29T17:41:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c2a078\u003e: \n resource group \"private-keyvault-mgxpmjvchvxg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mgxpmjvchvxg\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e9da9cc-6ed2-496f-a887-421a5555582f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e9da9cc-6ed2-496f-a887-421a5555582f\",\n \"name\": \"3e9da9cc-6ed2-496f-a887-421a5555582f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:41:56.828729728Z\",\n \"endTime\": \"2026-09-29T18:06:02.29902555Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q9dplvqkvipc2q8n5pplrfqn14c16 still exists (deletion dispatched at 2026-09-29T17:41:58Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3113c08422a61b88d138198c60a5baa89d197140e068dcb96e1353a8f7d06dba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"ef9b5a8a457501a78273f2be26c7ab62ca39007bc16f7ed67ee6a0623a2e4433","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0017f2228\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0017f2228\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"630622c6a57450abf346b078a2998ccd246f422c829fbee0489be2cfcccd545c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"220f32fff7042c724d79f6acdfa654917848098d7547032ecf17860b614ca368","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001e14168\u003e: \n resource group \"rg-candidate-4-20-w8n9ljjm626w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-5l8bh4\" in resourcegroup=\"rg-candidate-4-20-w8n9ljjm626w\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e18bb44-b0e5-444f-94b9-63d3a9ff27a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e18bb44-b0e5-444f-94b9-63d3a9ff27a9\",\n \"name\": \"9e18bb44-b0e5-444f-94b9-63d3a9ff27a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:10.150923038Z\",\n \"endTime\": \"2026-09-29T18:06:12.263898636Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8dfpl0ib2cthssvd6at5j30an4b9 still exists (deletion dispatched at 2026-09-29T17:42:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001e14168\u003e: \n resource group \"rg-candidate-4-20-w8n9ljjm626w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-5l8bh4\" in resourcegroup=\"rg-candidate-4-20-w8n9ljjm626w\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e18bb44-b0e5-444f-94b9-63d3a9ff27a9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9e18bb44-b0e5-444f-94b9-63d3a9ff27a9\",\n \"name\": \"9e18bb44-b0e5-444f-94b9-63d3a9ff27a9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:10.150923038Z\",\n \"endTime\": \"2026-09-29T18:06:12.263898636Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8dfpl0ib2cthssvd6at5j30an4b9 still exists (deletion dispatched at 2026-09-29T17:42:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6a3f9fcd1db35284dd05a8443b4b635cb9f4bd08009f68ac8605dde839d42328","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"alert","signature_id":"237b8d9f2894b8079c8797595bdd18c9dadbb45f0e18da236611709a9e83ed17","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T17:44:34Z\n Ended: 2026-09-29T18:19:33Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3617536-svc\", component=\"service-memory\", container=\"aro-hcp-frontend\", namespace=\"aro-hcp\", pod=\"aro-hcp-frontend-67d6bd86-dxk5c\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-frontend in pod aro-hcp-frontend-67d6bd86-dxk5c (namespace aro-hcp) on cluster ci01-j3617536-svc has averaged 177.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-29T17:44:34Z\n Ended: 2026-09-29T18:31:36Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3617536-svc\", component=\"service-memory\", container=\"aro-hcp-frontend\", namespace=\"aro-hcp\", pod=\"aro-hcp-frontend-67d6bd86-wwm6n\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-frontend in pod aro-hcp-frontend-67d6bd86-wwm6n (namespace aro-hcp) on cluster ci01-j3617536-svc has averaged 220.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"855f1b0879622e7a35f23dc20f9e79eec31cca36fa0b44863115bec2295a1f07","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"64b9781b9ebf4d6072288b494f80662542828d755239f88852e560beec0d4afe","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00259e030\u003e: \n resource group \"feature-aggregation-zkcnx8tdngds\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-zkcnx8tdngds\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8e3f22-69a7-45cb-9a4f-423ef0d53d3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8e3f22-69a7-45cb-9a4f-423ef0d53d3f\",\n \"name\": \"bd8e3f22-69a7-45cb-9a4f-423ef0d53d3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:53:02.426632555Z\",\n \"endTime\": \"2026-09-29T18:17:09.63164867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q9u7coc31q0vdncuqjcta1uk5qsbm still exists (deletion dispatched at 2026-09-29T17:53:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00259e030\u003e: \n resource group \"feature-aggregation-zkcnx8tdngds\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-zkcnx8tdngds\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8e3f22-69a7-45cb-9a4f-423ef0d53d3f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bd8e3f22-69a7-45cb-9a4f-423ef0d53d3f\",\n \"name\": \"bd8e3f22-69a7-45cb-9a4f-423ef0d53d3f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:53:02.426632555Z\",\n \"endTime\": \"2026-09-29T18:17:09.63164867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q9u7coc31q0vdncuqjcta1uk5qsbm still exists (deletion dispatched at 2026-09-29T17:53:04Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9006d1442b2c88f6be67c3d0d23cc841add3e74fc5339a4d9036ee857a99b3ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"cc65b08c1ed212f991bc652d7f4d72696d64f449b00f5533475ab93d813ea9ec","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001993998\u003e: \n resource group \"rg-candidate-5-0-md9fbphhxnnk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-fgxblj\" in resourcegroup=\"rg-candidate-5-0-md9fbphhxnnk\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebc781de-21cf-464a-bfc8-7070c532aca4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebc781de-21cf-464a-bfc8-7070c532aca4\",\n \"name\": \"ebc781de-21cf-464a-bfc8-7070c532aca4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:44.762057456Z\",\n \"endTime\": \"2026-09-29T18:06:52.248053651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8ffemtednb9jhms2juu0lkb65rap still exists (deletion dispatched at 2026-09-29T17:42:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001993998\u003e: \n resource group \"rg-candidate-5-0-md9fbphhxnnk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-fgxblj\" in resourcegroup=\"rg-candidate-5-0-md9fbphhxnnk\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebc781de-21cf-464a-bfc8-7070c532aca4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ebc781de-21cf-464a-bfc8-7070c532aca4\",\n \"name\": \"ebc781de-21cf-464a-bfc8-7070c532aca4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:44.762057456Z\",\n \"endTime\": \"2026-09-29T18:06:52.248053651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8ffemtednb9jhms2juu0lkb65rap still exists (deletion dispatched at 2026-09-29T17:42:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a64db3a5e4e05ad9104813ba457cd41bf6da017c945eb776f5a18a50e741b878","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"e91d886a2c61df0ca7479cf830bbcb4371436fb4d6bc0a84f559ea66902e725c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b724e0\u003e: \n resource group \"rg-candidate-4-21-zz5frflnrmk7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-rvbkqk\" in resourcegroup=\"rg-candidate-4-21-zz5frflnrmk7\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc3b32b-35a1-4cae-8747-5974cc8f58cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc3b32b-35a1-4cae-8747-5974cc8f58cb\",\n \"name\": \"8cc3b32b-35a1-4cae-8747-5974cc8f58cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:41:30.96598077Z\",\n \"endTime\": \"2026-09-29T18:05:32.361537862Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8gksdbb4mj4dub9bfiir7qb06n0b still exists (deletion dispatched at 2026-09-29T17:41:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b724e0\u003e: \n resource group \"rg-candidate-4-21-zz5frflnrmk7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-rvbkqk\" in resourcegroup=\"rg-candidate-4-21-zz5frflnrmk7\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc3b32b-35a1-4cae-8747-5974cc8f58cb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8cc3b32b-35a1-4cae-8747-5974cc8f58cb\",\n \"name\": \"8cc3b32b-35a1-4cae-8747-5974cc8f58cb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:41:30.96598077Z\",\n \"endTime\": \"2026-09-29T18:05:32.361537862Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8gksdbb4mj4dub9bfiir7qb06n0b still exists (deletion dispatched at 2026-09-29T17:41:33Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b476354addec9a076b6b120b15704d9b896edf7797db0ecfce4199415b48d506","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"252302ded4ddd125d8e8ddea82f70939951fc2b5247a0e95ae81ee43bb294acd","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001d822b8\u003e: \n resource group \"rg-candidate-5-1-4df2rb88kslv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-8lqd77\" in resourcegroup=\"rg-candidate-5-1-4df2rb88kslv\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f2dd559-18f1-4c7c-a090-7b2cb32f6352\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f2dd559-18f1-4c7c-a090-7b2cb32f6352\",\n \"name\": \"6f2dd559-18f1-4c7c-a090-7b2cb32f6352\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:25.116814076Z\",\n \"endTime\": \"2026-09-29T18:06:32.319934274Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8hfn91loc9a4dn47b48hi462fllh still exists (deletion dispatched at 2026-09-29T17:42:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001d822b8\u003e: \n resource group \"rg-candidate-5-1-4df2rb88kslv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-1-8lqd77\" in resourcegroup=\"rg-candidate-5-1-4df2rb88kslv\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f2dd559-18f1-4c7c-a090-7b2cb32f6352\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6f2dd559-18f1-4c7c-a090-7b2cb32f6352\",\n \"name\": \"6f2dd559-18f1-4c7c-a090-7b2cb32f6352\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:25.116814076Z\",\n \"endTime\": \"2026-09-29T18:06:32.319934274Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8hfn91loc9a4dn47b48hi462fllh still exists (deletion dispatched at 2026-09-29T17:42:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b596130f2889640a7328ed008a823cbf6273ae2defc518c9493432958d032ac9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"f9bd2121f9aea68deeeb566f8d59569ecd3d3addbbaade0c2a33c2b8bd7edc46","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f62228\u003e: \n resource group \"rg-hs-pre--59w8gdh7v4kd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-76lphh\" in resourcegroup=\"rg-hs-pre--59w8gdh7v4kd\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8f2838d-7c70-4eaa-a0a6-11f17ff5c319\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8f2838d-7c70-4eaa-a0a6-11f17ff5c319\",\n \"name\": \"e8f2838d-7c70-4eaa-a0a6-11f17ff5c319\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:40:59.703660294Z\",\n \"endTime\": \"2026-09-29T18:05:02.300476006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8fan7a4bv4voo6shcu8miu3t16sc still exists (deletion dispatched at 2026-09-29T17:41:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f62228\u003e: \n resource group \"rg-hs-pre--59w8gdh7v4kd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-76lphh\" in resourcegroup=\"rg-hs-pre--59w8gdh7v4kd\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8f2838d-7c70-4eaa-a0a6-11f17ff5c319\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e8f2838d-7c70-4eaa-a0a6-11f17ff5c319\",\n \"name\": \"e8f2838d-7c70-4eaa-a0a6-11f17ff5c319\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:40:59.703660294Z\",\n \"endTime\": \"2026-09-29T18:05:02.300476006Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8fan7a4bv4voo6shcu8miu3t16sc still exists (deletion dispatched at 2026-09-29T17:41:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c33089a67c37aa9cb5a2e35c46527c63dd463423c4b2a37201713c07885bd895","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"98d406871443eff302d384e85f097e58b4dc2873ac5049b1c98efba554398a68","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016523d8\u003e: \n resource group \"v20261001preview-wzffgbt2mwz4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-wzffgbt2mwz4\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8381eb45-2d3b-4d62-ae37-c77d242a81e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8381eb45-2d3b-4d62-ae37-c77d242a81e5\",\n \"name\": \"8381eb45-2d3b-4d62-ae37-c77d242a81e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:19.259624089Z\",\n \"endTime\": \"2026-09-29T18:06:22.287148437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8i0ta41gsi6sob46talt49qrms10 still exists (deletion dispatched at 2026-09-29T17:42:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0016523d8\u003e: \n resource group \"v20261001preview-wzffgbt2mwz4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-wzffgbt2mwz4\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8381eb45-2d3b-4d62-ae37-c77d242a81e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8381eb45-2d3b-4d62-ae37-c77d242a81e5\",\n \"name\": \"8381eb45-2d3b-4d62-ae37-c77d242a81e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:19.259624089Z\",\n \"endTime\": \"2026-09-29T18:06:22.287148437Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8i0ta41gsi6sob46talt49qrms10 still exists (deletion dispatched at 2026-09-29T17:42:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c9e1e0852a74ff90278ca344b01597ec1dbfc27ff62eb4b09d34c8bc08189fae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"f2cf322bdc7cc119702e40212065719729f9f239a9ccbe1432fb3029b16b99c9","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002ba60a8\u003e: \n resource group \"private-kas-fcp7wfpkhndv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-fcp7wfpkhndv\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bd8617f-eb80-42af-ad1d-cf8196c9ed2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bd8617f-eb80-42af-ad1d-cf8196c9ed2b\",\n \"name\": \"2bd8617f-eb80-42af-ad1d-cf8196c9ed2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:57.576170337Z\",\n \"endTime\": \"2026-09-29T18:07:02.239416479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8k9u1fl8ppmc0s9vg8obfj213vjh still exists (deletion dispatched at 2026-09-29T17:42:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002ba60a8\u003e: \n resource group \"private-kas-fcp7wfpkhndv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-fcp7wfpkhndv\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bd8617f-eb80-42af-ad1d-cf8196c9ed2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2bd8617f-eb80-42af-ad1d-cf8196c9ed2b\",\n \"name\": \"2bd8617f-eb80-42af-ad1d-cf8196c9ed2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:57.576170337Z\",\n \"endTime\": \"2026-09-29T18:07:02.239416479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8k9u1fl8ppmc0s9vg8obfj213vjh still exists (deletion dispatched at 2026-09-29T17:42:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d06ddaa252714b683368fe533d8d27f824ed5693822de91b7ba0e7246d3cd82f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"alert","signature_id":"bd0678e58ffc20c5a17a770813bd34e7fd461dd9be5c5d3f1ff3d65fc3c66bbd","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 22 time(s) (2 unknown, 20 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T18:06:16Z\n Ended: 2026-09-29T18:12:20Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3617536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4qlf9g4q7g15no702p9kevvbmidhl9-f6u1x0l2r8k7x5n\", pod=\"openshift-apiserver-5864fbdbd4-st9wn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4qlf9g4q7g15no702p9kevvbmidhl9-f6u1x0l2r8k7x5n/openshift-apiserver-5864fbdbd4-st9wn has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T18:12:19Z\n Ended: 2026-09-29T18:18:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3617536-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4q8gksdbb4mj4dub9bfiir7qb06n0b-e5v1y6s5y3h4c3s\", pod=\"azure-disk-csi-driver-controller-84b49bf957-24qh8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4q8gksdbb4mj4dub9bfiir7qb06n0b-e5v1y6s5y3h4c3s/azure-disk-csi-driver-controller-84b49bf957-24qh8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"dcaf18300bd578be1bbe8ae9ae60f632f5ea409981fb3c0ab539f2a4fa128bbb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"cc5326e09bb70c23eee58617238776b446e3bfea4ce6419761a6c3e563a570f6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001990360\u003e: \n resource group \"rg-candidate-4-23-bwspc6qf8blp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-w6tsln\" in resourcegroup=\"rg-candidate-4-23-bwspc6qf8blp\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b67fd1bc-0276-43ea-a23f-18874468dad4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b67fd1bc-0276-43ea-a23f-18874468dad4\",\n \"name\": \"b67fd1bc-0276-43ea-a23f-18874468dad4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:47.58421915Z\",\n \"endTime\": \"2026-09-29T18:06:52.227476407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8ici53q4og2b2j3j8ulfgiae6741 still exists (deletion dispatched at 2026-09-29T17:42:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001990360\u003e: \n resource group \"rg-candidate-4-23-bwspc6qf8blp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-w6tsln\" in resourcegroup=\"rg-candidate-4-23-bwspc6qf8blp\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b67fd1bc-0276-43ea-a23f-18874468dad4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b67fd1bc-0276-43ea-a23f-18874468dad4\",\n \"name\": \"b67fd1bc-0276-43ea-a23f-18874468dad4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:42:47.58421915Z\",\n \"endTime\": \"2026-09-29T18:06:52.227476407Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q8ici53q4og2b2j3j8ulfgiae6741 still exists (deletion dispatched at 2026-09-29T17:42:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e0c07370ac720cbebdc154787f14b2ccb9a9856f100cf0b0c8216220122cee5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104969648343617536","occurred_at":"2026-09-29T16:20:47Z","failed_at":"e2e","signature_id":"97a471caf0b3976649f801493084ea0a8b13cfa950f30f024bb7ec2959856f10","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f02318\u003e: \n resource group \"complex-cilium-kv-mfq6v52hmk8t\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mfq6v52hmk8t\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c1cbdb1-dc1a-4e08-a655-df076ccbaae2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c1cbdb1-dc1a-4e08-a655-df076ccbaae2\",\n \"name\": \"8c1cbdb1-dc1a-4e08-a655-df076ccbaae2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:43:08.623493087Z\",\n \"endTime\": \"2026-09-29T18:07:12.235576256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q98bpa7kaisvdgp52iprp7qgua195 still exists (deletion dispatched at 2026-09-29T17:43:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f02318\u003e: \n resource group \"complex-cilium-kv-mfq6v52hmk8t\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mfq6v52hmk8t\" to finish deleting: GET https://rp.j3617536.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c1cbdb1-dc1a-4e08-a655-df076ccbaae2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8c1cbdb1-dc1a-4e08-a655-df076ccbaae2\",\n \"name\": \"8c1cbdb1-dc1a-4e08-a655-df076ccbaae2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:43:08.623493087Z\",\n \"endTime\": \"2026-09-29T18:07:12.235576256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4q98bpa7kaisvdgp52iprp7qgua195 still exists (deletion dispatched at 2026-09-29T17:43:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104965212217020416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7124,"pr_state":"closed","pr_sha":"113526cb8f77445e2427551f69427f9bef948532","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T16:03:09Z","started_at":"2026-09-29T16:03:09Z","completed_at":"2026-09-29T16:12:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T16:21:17.627634723Z","timing_metadata_checked_at":"2026-09-29T16:21:17.627634723Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-29T16:21:17.630582215Z","region_metadata_checked_at":"2026-09-29T17:24:05.940484866Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"be4b61183ceca4d753b9881a74da6e571c468206c408dc77f710bdd4842ca137","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104965212217020416","occurred_at":"2026-09-29T16:03:09Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7245/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104960395847405568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7245,"pr_state":"closed","pr_sha":"f123239b74b59b29eef0693165ee8bff8c1e6522","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T15:44:01Z","started_at":"2026-09-29T15:44:01Z","completed_at":"2026-09-29T18:07:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T18:16:46.154718658Z","timing_metadata_checked_at":"2026-09-29T18:16:46.154718658Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T18:16:46.157223449Z","region_metadata_checked_at":"2026-09-29T18:16:46.157223449Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"21eb80ec70c060686a23d87a508ce3183db666c8854c466394d54b47aa9f3532","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7245/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104960395847405568","occurred_at":"2026-09-29T15:44:01Z","failed_at":"alert","signature_id":"7ec5f2dcf496dfee97dcd86257fcd8a980cd9fb11dcd2015be7c75f678b1aa6b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T16:53:04Z\n Ended: 2026-09-29T17:15:05Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7405568-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T16:53:04Z\n Ended: 2026-09-29T17:12:04Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7405568-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T16:53:04Z\n Ended: 2026-09-29T17:10:04Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7405568-svc\", component=\"backend\", name=\"clusterclusterserviceupdatedispatch\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterclusterserviceupdatedispatch has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104960350616031232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T15:43:50Z","started_at":"2026-09-29T15:43:50Z","completed_at":"2026-09-29T18:06:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T18:16:45.831869033Z","timing_metadata_checked_at":"2026-09-29T18:16:45.831869033Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T18:16:45.834857422Z","region_metadata_checked_at":"2026-09-29T18:16:45.834857422Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c84196842d1373b7705b200bfec2da756ca6b135260b55028b50749bc38641b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104960350616031232","occurred_at":"2026-09-29T15:43:50Z","failed_at":"e2e","signature_id":"7ba1ffd643ff08e0b7405369bab13e475ea063624297e7ce027fdf116b8dfd6a","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013d83f0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013d83f0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7231,"pr_state":"open","pr_sha":"0e517da494c4d64462fee111b4397e3569f04720","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T15:40:05Z","started_at":"2026-09-29T15:40:05Z","completed_at":"2026-09-29T18:05:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T18:16:45.853604554Z","timing_metadata_checked_at":"2026-09-29T18:16:45.853604554Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T18:16:45.856255044Z","region_metadata_checked_at":"2026-09-29T18:16:45.856255044Z"},"failed_at":["alert","e2e"],"failed_test_count":10,"occurrences":[{"row_id":"39eb0ef46f38c27532bd887bd73dda658993c320ce6fd52990b928a444994f84","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"e2e","signature_id":"9169589e2000a7897390f223955e473d2e2b91d9a64ed681a77eb22049a30636","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:151]: failed to verify cluster health or IngressController scope for cluster \"pi-4-20\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001273c98\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:151]: failed to verify cluster health or IngressController scope for cluster \"pi-4-20\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001273c98\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}},{"row_id":"40b01208f610a92a63e7619a2d1ba6cae585e0c5efc6df21c30698fd3a34db39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"alert","signature_id":"b5d04d1caac8b3fc08e30b6272c75260d3b44d703961b39c529a76f69663436e","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T16:54:05Z\n Ended: 2026-09-29T17:20:06Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/hcps-j0873728\", namespace=\"ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T16:54:05Z\n Ended: 2026-09-29T17:24:04Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/hcps-j0873728\", namespace=\"ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"51c811b61a18f3f9fd66ef95a251fcb658b624cefa70566b0a32ddd9221ab052","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"alert","signature_id":"9f956f7b3d8ed75ae06dd8ff4cbabe40730ca3e664b0899f988f4947ed01fe2e","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T16:58:29Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-29T16:58:29Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T17:07:31Z\n Ended: 2026-09-29T17:35:30Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"klusterlet-2t4pk8k1eqei6ipqjn5eo2qde37f917q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4pk8k1eqei6ipqjn5eo2qde37f917q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-29T17:07:31Z\n Ended: 2026-09-29T17:35:30Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"klusterlet-2t4pk8k1eqei6ipqjn5eo2qde37f917q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4pk8k1eqei6ipqjn5eo2qde37f917q/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-29T17:29:29Z\n Ended: 2026-09-29T17:39:29Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"klusterlet-2t4pvsjcuds2fgpd5nc13jc1b6vtv0ra\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4pvsjcuds2fgpd5nc13jc1b6vtv0ra/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-29T17:29:29Z\n Ended: 2026-09-29T17:39:29Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"klusterlet-2t4pvsjcuds2fgpd5nc13jc1b6vtv0ra\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4pvsjcuds2fgpd5nc13jc1b6vtv0ra/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-29T17:34:30Z\n Ended: 2026-09-29T17:40:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"klusterlet-2t4q266hvr5t8e6kmj10ip38mft9e346\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4q266hvr5t8e6kmj10ip38mft9e346/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-29T17:34:30Z\n Ended: 2026-09-29T17:41:29Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"klusterlet-2t4q266hvr5t8e6kmj10ip38mft9e346\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4q266hvr5t8e6kmj10ip38mft9e346/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"5d3efc12d640986a83d5c0524ccd338c27a65bca51767334df31cc20f10ea508","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"alert","signature_id":"0d0a808954e147f0a519c7495b00af821e09439035296652cb563d2d6d584b13","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T17:08:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-29T17:08:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T17:17:28Z\n Ended: 2026-09-29T17:35:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"klusterlet-2t4pk8k1eqei6ipqjn5eo2qde37f917q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment klusterlet-2t4pk8k1eqei6ipqjn5eo2qde37f917q/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-29T17:17:28Z\n Ended: 2026-09-29T17:35:30Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/services-j0873728\", namespace=\"klusterlet-2t4pk8k1eqei6ipqjn5eo2qde37f917q\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment klusterlet-2t4pk8k1eqei6ipqjn5eo2qde37f917q/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"6e286533b8568fd90c210936fc1ff28396bcb238cf2e8c9aada9fc744c86176c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"e2e","signature_id":"77e382bd18c425a1486a48748186b9b6712b93887e949de044f06f7c408107c7","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:151]: failed to verify cluster health or IngressController scope for cluster \"pi-4-21\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00152e078\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:151]: failed to verify cluster health or IngressController scope for cluster \"pi-4-21\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00152e078\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}},{"row_id":"756dc7e09648042d8766e6fdc84c7cc5b3ed6bc21a2481f1c46f3b061ee8f50c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"e2e","signature_id":"5665ea195025c891d95ae7b5fda7a67f14d48f4ae765d9b9f5916cf5cac18306","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-r85k6z\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001224f00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-r85k6z: failed waiting for cluster=\"cp-ystream-upgrade-4-23-r85k6z\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-v5dv9ks6kpb7\" to finish creating: GET https://rp.j0873728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f6f7bc4-1ea5-4377-a861-1080166398f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f6f7bc4-1ea5-4377-a861-1080166398f9\",\n \"name\": \"1f6f7bc4-1ea5-4377-a861-1080166398f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T16:32:43.227956369Z\",\n \"endTime\": \"2026-09-29T17:01:49.778635843Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-r85k6z\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001224f00\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-r85k6z: failed waiting for cluster=\"cp-ystream-upgrade-4-23-r85k6z\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-v5dv9ks6kpb7\" to finish creating: GET https://rp.j0873728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f6f7bc4-1ea5-4377-a861-1080166398f9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f6f7bc4-1ea5-4377-a861-1080166398f9\",\n \"name\": \"1f6f7bc4-1ea5-4377-a861-1080166398f9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T16:32:43.227956369Z\",\n \"endTime\": \"2026-09-29T17:01:49.778635843Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"76791d6bdb0f19f2100ceb0d7399d33eb177150d440d5901076ef3ef34a41486","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"alert","signature_id":"976371b4017abed28e98154a08cffbe3fe85ee156f266de970ebd6e15b9b14a8","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T16:55:04Z\n Ended: 2026-09-29T17:23:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/hcps-j0873728\", namespace=\"ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"77d06f7d-9903-4077-b53b-f78669a34cec\"\n Description: Pod ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-29T16:55:04Z\n Ended: 2026-09-29T17:20:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/hcps-j0873728\", namespace=\"ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"77d06f7d-9903-4077-b53b-f78669a34cec\"\n Description: Pod ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"7ad635b4068c28ea2fbcc8df1e3fd847ef9cd1c6f212f09f96485ea8baad23d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"alert","signature_id":"5baf42fb9cff111fb8842d9a90fd2e83de6d6e7af04a3aed4105be3c8269d5ae","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T16:33:27Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-78648d9c8d-gd726\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-78648d9c8d-gd726 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"b85b26e085ddc97473a72dd42e4e253b305fca2aee5efabd66164a06e6d075cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"e2e","signature_id":"918eb084718334945463e39eb51d690600b1e9a41ea0bb3b240a9f3fc995dac8","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00097d230\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00097d230\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"e2943862214f200d87142ba691fe04fa98c0868fad8aef9ff1e6c3f9566b76a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104959408780873728","occurred_at":"2026-09-29T15:40:05Z","failed_at":"alert","signature_id":"52c71cca1c0599f2a47994c3a793406e5c5a2c97fd6aa0bc7fe0d144e4ae08df","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T17:08:04Z\n Ended: 2026-09-29T17:26:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0873728-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.132:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0873728/providers/microsoft.monitor/accounts/hcps-j0873728\", namespace=\"ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-qqn8d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4pkau0b0nuc1epds7jnuhhosgt2got-c8u2v9v3j9q8t4s/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104953578396323840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7244,"pr_state":"closed","pr_sha":"cbddf39100b6fb0f90835d16a76eb4cb925cc7df","final_merged_sha":"cbddf39100b6fb0f90835d16a76eb4cb925cc7df","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T15:16:48Z","started_at":"2026-09-29T15:16:48Z","completed_at":"2026-09-29T17:24:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T17:44:34.691283274Z","timing_metadata_checked_at":"2026-09-29T17:44:34.691283274Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T17:44:34.693936365Z","region_metadata_checked_at":"2026-09-29T17:44:34.693936365Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fb2a4ae8f641f8e894ee370e9c58be890a00e736b8160ae71e86edf5ba29bf53","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7244/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104953578396323840","occurred_at":"2026-09-29T15:16:48Z","failed_at":"e2e","signature_id":"ea4c67dab9692940dc7eae96d5f5a8287597494b0b56fb247045f864dadf3f94","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00180d170\u003e: \n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00180d170\u003e: \n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"769393839c5169271584454d33115d1d7833ffb7d824c870abf6a086299a40b3","failure_pattern":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5","search_query":"VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["769393839c5169271584454d33115d1d7833ffb7d824c870abf6a086299a40b3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104951367071174656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"7cc3bc8a1c5939f7125d9af7d41f712d1e116e4e","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T15:08:08Z","started_at":"2026-09-29T15:08:08Z","completed_at":"2026-09-29T17:35:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T17:44:34.323301735Z","timing_metadata_checked_at":"2026-09-29T17:44:34.323301735Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T17:44:34.326173826Z","region_metadata_checked_at":"2026-09-29T17:44:34.326173826Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f1226a8dcf38666eebc359cb9b5c599d3a054f91ae61f05172af38c73c4f8838","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104951367071174656","occurred_at":"2026-09-29T15:08:08Z","failed_at":"e2e","signature_id":"9cc26b07047aec4d09a21eb6bcaa1c7dca5f135cd11ef9a2aedb3fadeff1cfd8","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013a3548\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013a3548\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104951111046664192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"closed","pr_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","final_merged_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T15:07:02Z","started_at":"2026-09-29T15:07:02Z","completed_at":"2026-09-29T17:32:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T17:44:34.066586245Z","timing_metadata_checked_at":"2026-09-29T17:44:34.066586245Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T17:44:34.07451782Z","region_metadata_checked_at":"2026-09-29T17:44:34.07451782Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"bf725f5fa2cff988ec3136de3db5e1f55f34b22112fe076f803aeb58593e5f9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104951111046664192","occurred_at":"2026-09-29T15:07:02Z","failed_at":"e2e","signature_id":"7a10f11f9b11b3d03e506627f0736734e96480fdb6cb4d5dc6be9bb18ed4e98c","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0016143f0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0016143f0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104950948592881664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"288fff3af439c6b5a15a853a8595a3f43423e44a","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T15:06:28Z","started_at":"2026-09-29T15:06:28Z","completed_at":"2026-09-29T17:33:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T17:44:34.459868304Z","timing_metadata_checked_at":"2026-09-29T17:44:34.459868304Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T17:44:34.462640595Z","region_metadata_checked_at":"2026-09-29T17:44:34.462640595Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"aeab55ce113272cf3b59dfd113550af93d4551f9b154d5c81504761929dc3396","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104950948592881664","occurred_at":"2026-09-29T15:06:28Z","failed_at":"e2e","signature_id":"9e94a725b1e01d264bfbf0329554f29d9fff7ff5387c58da01db5baac4f9af90","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000fdb1b8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000fdb1b8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104949753673420800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"8470e4a8152fec1ad8e98ecdb32250dd598096a9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T15:01:37Z","started_at":"2026-09-29T15:01:37Z","completed_at":"2026-09-29T17:24:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T17:44:34.200502622Z","timing_metadata_checked_at":"2026-09-29T17:44:34.200502622Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T17:44:34.203337113Z","region_metadata_checked_at":"2026-09-29T17:44:34.203337113Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"02b9955a8d6069f0b6b6251b18d7663cf25fc92d3f61693080e86d075cdb8070","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104949753673420800","occurred_at":"2026-09-29T15:01:37Z","failed_at":"e2e","signature_id":"0d03c7f2f190490d25af59b262092e32a63ce8e6b412a7c46c1edbd07e983b56","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001274918\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001274918\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104949577500069888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7183,"pr_state":"closed","pr_sha":"30a9a5e8593d1558577c9c639b734ef2df1555b8","final_merged_sha":"30a9a5e8593d1558577c9c639b734ef2df1555b8","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T15:00:33Z","started_at":"2026-09-29T15:00:33Z","completed_at":"2026-09-29T17:18:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T17:24:06.224931647Z","timing_metadata_checked_at":"2026-09-29T17:24:06.224931647Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T17:24:06.228056039Z","region_metadata_checked_at":"2026-09-29T17:24:06.228056039Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"cc1772f6722da5ee0b75450469bc7be1eb08b0a1084b36c3ad269fc550fd9977","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104949577500069888","occurred_at":"2026-09-29T15:00:33Z","failed_at":"e2e","signature_id":"f883de1484fb937d06920a3faee39bf43f48090bba8c3671addfed0e67bc977a","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00135a660\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.007s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00135a660\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"ec73bbae82533684960e211d0bf61ca455463c1ca3fe56dcf1367ef844ff28e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7183/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104949577500069888","occurred_at":"2026-09-29T15:00:33Z","failed_at":"alert","signature_id":"aeb0e7cf0f7fb6668a74127ede7587f34beff19ae9034d243e6f7865af3d860f","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T16:09:31Z\n Ended: 2026-09-29T16:27:31Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0069888-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T16:09:31Z\n Ended: 2026-09-29T16:25:30Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0069888-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104948570607063040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"0bdd36fc9bed8d937676c8592b848cd9cbcde70d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T14:57:00Z","started_at":"2026-09-29T14:57:00Z","completed_at":"2026-09-29T17:22:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T17:44:33.880032433Z","timing_metadata_checked_at":"2026-09-29T17:44:33.880032433Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T17:44:33.883406823Z","region_metadata_checked_at":"2026-09-29T17:44:33.883406823Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"06f252e96e9043a42cef87140c9b4a60ae16948109e27e3d69d3608c52437947","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104948570607063040","occurred_at":"2026-09-29T14:57:00Z","failed_at":"e2e","signature_id":"389b7b9250b2eaa107eca96a99538d0a359f7f9a8406c6fde7836da2767c837c","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000c2f680\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000c2f680\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"c5ee287900f470987e4293fbef4a02972cb8a1db83e05f3f8b025c5163d20f99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104948570607063040","occurred_at":"2026-09-29T14:57:00Z","failed_at":"e2e","signature_id":"7a309032f1a2720a45ffb02f37aad110798716aa7ba025c5cf0d8e67390fa0cd","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:282]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001519d40\u003e: \n GET https://rp.j7063040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/314c81fe-2478-47d0-ae67-1f8bdc8c0f20\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/314c81fe-2478-47d0-ae67-1f8bdc8c0f20' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/314c81fe-2478-47d0-ae67-1f8bdc8c0f20\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:282]: failed to update cluster \"agg-cluster\" authorized CIDRs to include test runner IP\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001519d40\u003e: \n GET https://rp.j7063040.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/314c81fe-2478-47d0-ae67-1f8bdc8c0f20\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/314c81fe-2478-47d0-ae67-1f8bdc8c0f20' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/314c81fe-2478-47d0-ae67-1f8bdc8c0f20\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104937201925099520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T14:11:51Z","started_at":"2026-09-29T14:11:51Z","completed_at":"2026-09-29T16:28:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T16:31:54.41508646Z","timing_metadata_checked_at":"2026-09-29T16:31:54.41508646Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T16:31:54.418398351Z","region_metadata_checked_at":"2026-09-29T16:31:54.418398351Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"094d2f447aa8713ec6eb7f9ae00b5d407bc52364d51950ac4b3c80a9b89cc11e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104937201925099520","occurred_at":"2026-09-29T14:11:51Z","failed_at":"e2e","signature_id":"f383c645a03f6826e44aea671a57dd213a586f39474f1175dbeef4bd16179d39","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ba0c90\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000ba0c90\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"889274360a6e7973c2bd508dace41e8f9f4079f30d3e0c284fcb081ac9cc8547","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104937201925099520","occurred_at":"2026-09-29T14:11:51Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T15:17:51Z\n Ended: 2026-09-29T15:33:51Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5099520-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"d2d8d917dd41575f15fcb259aeff40678f780e52d57d622f965e8f5ad3a6e759","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104937201925099520","occurred_at":"2026-09-29T14:11:51Z","failed_at":"e2e","signature_id":"33887c32d252b866727346e81c84fde74866ad102c6552f6761977e0a5bfe0ff","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fccf00\u003e: \n failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-01-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fccf00\u003e: \n failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-01-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 2 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104936582908743680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7104,"pr_state":"closed","pr_sha":"96fefec7020ace2443c32f7f522239a74f96b4d2","final_merged_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T14:09:23Z","started_at":"2026-09-29T14:09:23Z","completed_at":"2026-09-29T16:39:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T16:53:34.584639254Z","timing_metadata_checked_at":"2026-09-29T16:53:34.584639254Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T16:53:34.590137438Z","region_metadata_checked_at":"2026-09-29T16:53:34.590137438Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"cf4f869ba75f37134514427cedce1610b21358fa4d54b851f552912d849b3848","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104936582908743680","occurred_at":"2026-09-29T14:09:23Z","failed_at":"e2e","signature_id":"67b212a1fc4909ceb93fb9a6f502055d6a666ef203f42b4161c10ffb12d5d1ec","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013bd518\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013bd518\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104936554697854976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"closed","pr_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","final_merged_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T14:09:17Z","started_at":"2026-09-29T14:09:17Z","completed_at":"2026-09-29T14:41:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T14:46:14.540057179Z","timing_metadata_checked_at":"2026-09-29T14:46:14.540057179Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T14:46:14.543477971Z","region_metadata_checked_at":"2026-09-29T14:46:14.543477971Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"1a41860dd2a11ba28d56ac9703216f6f079e73bfbbcd90428dc39df4838eec80","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104936554697854976","occurred_at":"2026-09-29T14:09:17Z","failed_at":"alert","signature_id":"25a286c528ab9b1eb1f357fc541983b6a2afe706d21b4a2ebca1ed7259de7625","test_name":"[aro-hcp-observability] [svc] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch svc alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-yt' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"db3f2570b04dae06bc06d8e34fe57270830b746e20e2d04146f404b7f2a3e0a7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104936554697854976","occurred_at":"2026-09-29T14:09:17Z","failed_at":"alert","signature_id":"089ef16f9f72bfc6a0703db0194cf1705d877b51575aef7422f3ea39575bc742","test_name":"[aro-hcp-observability] [hcp] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch hcp alert rules: failed to list prometheus rule groups: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt/providers/Microsoft.AlertsManagement/prometheusRuleGroups\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-yt' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.AlertsManagement","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"e032c4a10d90b7f565183c4931f44614182768be6a8fb783d8efed2df66fbfce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104936554697854976","occurred_at":"2026-09-29T14:09:17Z","failed_at":"alert","signature_id":"96aed78acfd2f056cab2a6a2653ad3e5b9697c24a19249b6dfd060bc03b574eb","test_name":"[aro-hcp-observability] [infra] alert collection is complete","test_suite":"aro-hcp-tests","failure_text":"alert collection incomplete\nfailed to fetch metric alert rules for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt: failed to list metric alert rules: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-yt/providers/Microsoft.Insights/metricAlerts\n--------------------------------------------------------------------------------\nRESPONSE 404: 404 Not Found\nERROR CODE: ResourceGroupNotFound\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-yt' could not be found.\"\n }\n}\n--------------------------------------------------------------------------------","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Insights","search_query":"ERROR CODE: ResourceGroupNotFound"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["2f6a5d821e7ca555fe32df7ee1edd93fc21c73e16f1e67a96805aaee3bcd51d5","35b44a6b9ab05412ceadb04a097872830c7c851f6a1a3193bcd5bd522c51b4f6"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104930620709801984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"a1395f1c08d6ec9f8dfd9d856318948882219e04","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T13:45:42Z","started_at":"2026-09-29T13:45:42Z","completed_at":"2026-09-29T16:03:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T16:10:32.234992145Z","timing_metadata_checked_at":"2026-09-29T16:10:32.234992145Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T16:10:32.237434437Z","region_metadata_checked_at":"2026-09-29T16:10:32.237434437Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"27c691505bf02a6912c869a29152cfbc5c4be6d62fc0878bfe48eb5c8da55f50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104930620709801984","occurred_at":"2026-09-29T13:45:42Z","failed_at":"e2e","signature_id":"3dc17822630f258848b82f72daa2a597d2082499b9f50d46adc566caa1760694","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000d1e2e8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000d1e2e8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"6df2ba0766418e7e26612eea1e36278f47b99d8234db61d8b9beaf9973e108f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104930620709801984","occurred_at":"2026-09-29T13:45:42Z","failed_at":"e2e","signature_id":"91b598d93817d526c7507d077770deac8e1acce827ccf38f2eac15ad9eaa4a8d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xzhqxr\" for cluster \"cluster-candidate-5-1-xzhqxr\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ba760\u003e: \n failed to create NodePool np-xzhqxr: failed waiting for nodepool=\"np-xzhqxr\" for cluster \"cluster-candidate-5-1-xzhqxr\" in resourcegroup=\"rg-candidate-5-1-sssjm2njvf46\" to finish creating: GET https://rp.j9801984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21054da5-6b20-4602-a074-747c4c269359\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21054da5-6b20-4602-a074-747c4c269359\",\n \"name\": \"21054da5-6b20-4602-a074-747c4c269359\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T14:50:45.293305718Z\",\n \"endTime\": \"2026-09-29T15:09:47.445356322Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4s7q0x9g8u2v1p-np-xzhqxr-hxgkn-mjgcb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-sssjm2njvf46-candidate-xzhqxr--managed/providers/Microsoft.Compute/virtualMachines/l4s7q0x9g8u2v1p-np-xzhqxr-hxgkn-mjgcb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4s7q0x9g8u2v1p-np-xzhqxr-hxgkn-mjgcb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-sssjm2njvf46-candidate-xzhqxr--managed/providers/Microsoft.Compute/virtualMachines/l4s7q0x9g8u2v1p-np-xzhqxr-hxgkn-mjgcb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-xzhqxr\" for cluster \"cluster-candidate-5-1-xzhqxr\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011ba760\u003e: \n failed to create NodePool np-xzhqxr: failed waiting for nodepool=\"np-xzhqxr\" for cluster \"cluster-candidate-5-1-xzhqxr\" in resourcegroup=\"rg-candidate-5-1-sssjm2njvf46\" to finish creating: GET https://rp.j9801984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21054da5-6b20-4602-a074-747c4c269359\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/21054da5-6b20-4602-a074-747c4c269359\",\n \"name\": \"21054da5-6b20-4602-a074-747c4c269359\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T14:50:45.293305718Z\",\n \"endTime\": \"2026-09-29T15:09:47.445356322Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l4s7q0x9g8u2v1p-np-xzhqxr-hxgkn-mjgcb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-sssjm2njvf46-candidate-xzhqxr--managed/providers/Microsoft.Compute/virtualMachines/l4s7q0x9g8u2v1p-np-xzhqxr-hxgkn-mjgcb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l4s7q0x9g8u2v1p-np-xzhqxr-hxgkn-mjgcb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-sssjm2njvf46-candidate-xzhqxr--managed/providers/Microsoft.Compute/virtualMachines/l4s7q0x9g8u2v1p-np-xzhqxr-hxgkn-mjgcb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104929398925824000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7241,"pr_state":"closed","pr_sha":"ee4479e302079878c9a9758a921ca483cadac619","final_merged_sha":"ee4479e302079878c9a9758a921ca483cadac619","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T13:40:51Z","started_at":"2026-09-29T13:40:51Z","completed_at":"2026-09-29T15:54:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T15:59:49.751965043Z","timing_metadata_checked_at":"2026-09-29T15:59:49.751965043Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T15:59:49.754900734Z","region_metadata_checked_at":"2026-09-29T15:59:49.754900734Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3848fb8677e832c64349a49860ed480cee900004726dce3aafe4005d223c0b88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104929398925824000","occurred_at":"2026-09-29T13:40:51Z","failed_at":"e2e","signature_id":"d3c0a2be9aca79665bfb7948a3ba64e7c469472725fec9a17990548ca40994fb","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001445230\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001445230\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"ee2c61032cf2a44129588bcfe2192d196318c86916a70d294cafa11ca3bba22e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7241/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104929398925824000","occurred_at":"2026-09-29T13:40:51Z","failed_at":"e2e","signature_id":"11d6ab32fb0eee8d6a3f26fec82de02fff4351673cc90f0ed7eb799874caf5b1","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0008aa800\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p7k2xh5g9wv9\" to finish creating: GET https://rp.j5824000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be0a1eb9-b1b2-4a75-abe0-ce39a79c7bde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be0a1eb9-b1b2-4a75-abe0-ce39a79c7bde\",\n \"name\": \"be0a1eb9-b1b2-4a75-abe0-ce39a79c7bde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T14:34:49.184621543Z\",\n \"endTime\": \"2026-09-29T14:53:52.314346782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a0k9y7c0o7e2r3d-np-delete-main-m2wn4-jg6rj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-p7k2xh5g9wv9--managed/providers/Microsoft.Compute/virtualMachines/a0k9y7c0o7e2r3d-np-delete-main-m2wn4-jg6rj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a0k9y7c0o7e2r3d-np-delete-main-m2wn4-jg6rj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-p7k2xh5g9wv9--managed/providers/Microsoft.Compute/virtualMachines/a0k9y7c0o7e2r3d-np-delete-main-m2wn4-jg6rj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0008aa800\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-p7k2xh5g9wv9\" to finish creating: GET https://rp.j5824000.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be0a1eb9-b1b2-4a75-abe0-ce39a79c7bde\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/be0a1eb9-b1b2-4a75-abe0-ce39a79c7bde\",\n \"name\": \"be0a1eb9-b1b2-4a75-abe0-ce39a79c7bde\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T14:34:49.184621543Z\",\n \"endTime\": \"2026-09-29T14:53:52.314346782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a0k9y7c0o7e2r3d-np-delete-main-m2wn4-jg6rj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-p7k2xh5g9wv9--managed/providers/Microsoft.Compute/virtualMachines/a0k9y7c0o7e2r3d-np-delete-main-m2wn4-jg6rj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a0k9y7c0o7e2r3d-np-delete-main-m2wn4-jg6rj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-p7k2xh5g9wv9--managed/providers/Microsoft.Compute/virtualMachines/a0k9y7c0o7e2r3d-np-delete-main-m2wn4-jg6rj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-mvqbmn55cv5b\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104929253865820160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"closed","pr_sha":"d1e472cd237fcf0745e383a11147953c3bd79e5c","final_merged_sha":"ea2df83db19482a983a29965803a71b1f8158ab7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T13:40:16Z","started_at":"2026-09-29T13:40:16Z","completed_at":"2026-09-29T14:19:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T14:25:01.67402867Z","timing_metadata_checked_at":"2026-09-29T14:25:01.67402867Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T14:25:01.676922561Z","region_metadata_checked_at":"2026-09-29T14:25:01.676922561Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"32d162f545488b62dcbed55af03af26cf437f8b07e6f88348bdec51ae14f7e70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104929253865820160","occurred_at":"2026-09-29T13:40:16Z","failed_at":"provision","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","test_suite":"templatize-pipeline","failure_text":"time=2026-09-29T14:11:12.980Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n Kind: ImageMirror\\n From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-29T14:11:56.710Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca to arohcpsvcdev.azurecr.io/nshneor/cs:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca.\\nThe image will still be available under it's original digest sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca in the target registry.\\nError: failed to resolve sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca arohcpsvcdev.azurecr.io/nshneor/cs:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca --from-registry-config /tmp/tmp.BCcaJ3bXs0/containers/auth.json --to-registry-config /tmp/tmp.BCcaJ3bXs0/containers/auth.json\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a","failure_pattern":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/nshneor/cs:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json","search_query":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca arohcpsvcdev.azurecr.io/nshneor/cs:a18a76b7071862c8f5234d49c9c3e5ac12659f2d8e31792c240e0f4848f81bca --from-registry-config /tmp/tmp.BCcaJ3bXs0/containers/auth.json --to-registry-config /tmp/tmp.BCcaJ3bXs0/containers/auth.json"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104928338156982272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"b31d1beb59bff387590cbcbf9e6a52530a67f89b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T13:36:12Z","started_at":"2026-09-29T13:36:12Z","completed_at":"2026-09-29T13:50:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T13:54:24.706222215Z","timing_metadata_checked_at":"2026-09-29T13:54:24.706222215Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-29T13:54:24.709002204Z","region_metadata_checked_at":"2026-09-29T14:56:55.253312736Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5d403037de045697636c47b8fdbd145c297b94f7f13a39346220a2b0421cc1e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104928338156982272","occurred_at":"2026-09-29T13:36:12Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104926229218988032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6141,"pr_state":"open","pr_sha":"e7317782f41b5555b1967bb298992bb4e4f3d31e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T13:27:39Z","started_at":"2026-09-29T13:27:39Z","completed_at":"2026-09-29T14:48:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T14:56:55.154260769Z","timing_metadata_checked_at":"2026-09-29T14:56:55.154260769Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T14:56:55.15688546Z","region_metadata_checked_at":"2026-09-29T14:56:55.15688546Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"eca4a0a9f7510d36add59b3bd4b4606e549808e256bb7847d07f12ad5830f87b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104926229218988032","occurred_at":"2026-09-29T13:27:39Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104919343220396032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7003,"pr_state":"closed","pr_sha":"b99db5f92f8185be272f4a98aa4b33b0491a69cd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T13:00:30Z","started_at":"2026-09-29T13:00:30Z","completed_at":"2026-09-29T13:16:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T13:33:11.105994633Z","timing_metadata_checked_at":"2026-09-29T13:33:11.105994633Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-29T13:33:11.109659821Z","region_metadata_checked_at":"2026-09-29T14:35:37.584158919Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ade85cfd25e02085d887fb696c2f1caaf1a44ba8e6efca15650a4908eda1589a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7003/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104919343220396032","occurred_at":"2026-09-29T13:00:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7005/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104917356470865920","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7005,"pr_state":"open","pr_sha":"2385fca99af2de43e8696b1005fdbd6d487c6d56","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T12:52:59Z","started_at":"2026-09-29T12:52:59Z","completed_at":"2026-09-29T13:30:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T13:43:53.858860977Z","timing_metadata_checked_at":"2026-09-29T13:43:53.858860977Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-29T13:43:53.862183668Z","region_metadata_checked_at":"2026-09-29T14:46:14.624632162Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f36cfb0901631e3bd319a491ff531a3d7673433ee0b092ad90cde1f7d0bffa87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7005/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104917356470865920","occurred_at":"2026-09-29T12:52:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104916576108023808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7220,"pr_state":"open","pr_sha":"540856c435b752648f8f78fe4a157f67de6eed0a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T12:49:46Z","started_at":"2026-09-29T12:49:46Z","completed_at":"2026-09-29T15:16:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T15:17:54.638685693Z","timing_metadata_checked_at":"2026-09-29T15:17:54.638685693Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T15:17:54.641170585Z","region_metadata_checked_at":"2026-09-29T15:17:54.641170585Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9ae80ae8f7c5edac776558dcbbf5fec274a63c01cdcabae1a9770c1ef6152b22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104916576108023808","occurred_at":"2026-09-29T12:49:46Z","failed_at":"e2e","signature_id":"91c4f7a7e76f6b9a4589dce76473fd679df12120d5f02312da7d5579f5bba6bd","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00175b038\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00175b038\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"f6ada59880b6b7418a07fa2df8b4b8df7db47d43776d7916489e98c0720eab0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7220/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104916576108023808","occurred_at":"2026-09-29T12:49:46Z","failed_at":"e2e","signature_id":"114a766528263895189731e19a59182a9052143c7a2913843f40763ee856f10f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-44q48s9f9dgr/cluster-candidate-4-20-7527zj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc280\u003e: \n failed to create HCP cluster cluster-candidate-4-20-7527zj: failed waiting for cluster=\"cluster-candidate-4-20-7527zj\" in resourcegroup=\"rg-candidate-4-20-44q48s9f9dgr\" to finish creating: GET https://rp.j8023808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ff3306c-ab44-4063-a7ff-2b4768cdd9cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ff3306c-ab44-4063-a7ff-2b4768cdd9cd\",\n \"name\": \"0ff3306c-ab44-4063-a7ff-2b4768cdd9cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T13:49:46.872625021Z\",\n \"endTime\": \"2026-09-29T14:18:55.212684595Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.40 is Partial (want Completed), started 21m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-44q48s9f9dgr/cluster-candidate-4-20-7527zj should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc280\u003e: \n failed to create HCP cluster cluster-candidate-4-20-7527zj: failed waiting for cluster=\"cluster-candidate-4-20-7527zj\" in resourcegroup=\"rg-candidate-4-20-44q48s9f9dgr\" to finish creating: GET https://rp.j8023808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ff3306c-ab44-4063-a7ff-2b4768cdd9cd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0ff3306c-ab44-4063-a7ff-2b4768cdd9cd\",\n \"name\": \"0ff3306c-ab44-4063-a7ff-2b4768cdd9cd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T13:49:46.872625021Z\",\n \"endTime\": \"2026-09-29T14:18:55.212684595Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.40 is Partial (want Completed), started 21m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104916338194518016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7071,"pr_state":"open","pr_sha":"6e66d4c793650a413acda3f74dda6a03700e788a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T12:48:57Z","started_at":"2026-09-29T12:48:57Z","completed_at":"2026-09-29T15:19:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T15:28:08.017792257Z","timing_metadata_checked_at":"2026-09-29T15:28:08.017792257Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T15:28:08.021473345Z","region_metadata_checked_at":"2026-09-29T15:28:08.021473345Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"ce2c48c9113549a08a91c8e42fb996a7bd2f4ae9f7931be07627c69245d8732a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104916338194518016","occurred_at":"2026-09-29T12:48:57Z","failed_at":"e2e","signature_id":"6494998e2c8ff0ad84b037bd741421e850ad915b7047909c020d13e76285ab82","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000df54d0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000df54d0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104915824979480576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7238,"pr_state":"closed","pr_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","final_merged_sha":"6a1ffdb2da1b44e3e97762eec9c6e4bac7581c86","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T12:46:54Z","started_at":"2026-09-29T12:46:54Z","completed_at":"2026-09-29T15:21:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T15:28:08.21721331Z","timing_metadata_checked_at":"2026-09-29T15:28:08.21721331Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T15:28:08.219862202Z","region_metadata_checked_at":"2026-09-29T15:28:08.219862202Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"61dc5620209320e70af8c9a629c2d332e0cb405fe8b2db3d3129778fff6eda4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7238/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104915824979480576","occurred_at":"2026-09-29T12:46:54Z","failed_at":"e2e","signature_id":"904042afeae014aad04a6bff53e1f8ca8afad42f8455b7f95bb5a5756a4ddb13","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0000132a8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0000132a8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104901125449715712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6141,"pr_state":"open","pr_sha":"d8a7da238df7b9e3f1b61100ffdda62613457603","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T11:48:30Z","started_at":"2026-09-29T11:48:30Z","completed_at":"2026-09-29T12:45:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T13:00:59.238283289Z","timing_metadata_checked_at":"2026-09-29T13:00:59.238283289Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T13:00:59.241261779Z","region_metadata_checked_at":"2026-09-29T13:00:59.241261779Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"23980616a388abfe6987b90b4066ff8c22353149ec9b58fb7a3d8b662763e7d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104901125449715712","occurred_at":"2026-09-29T11:48:30Z","failed_at":"provision","signature_id":"d61d44b9c7c7b88745261b45ecc7eca7267d6c81a06671945dc410cb687146ca","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-09-29T12:22:57.834Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-29T12:22:58.550Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy values=\"map[azureArmHelperIdentityCertName:armHelperPoolCert-31 azureArmHelperIdentityClientId:15d9575a-485d-4e5f-9c06-003e8353383b azureArmHelperMockFpaPrincipalId:47f69502-0065-4d9a-b19b-d403e183d2f4 azureCloudEnvironmentName:AzurePublicCloud azureCsMiClientId:79ab0919-0aa8-453d-80c5-59ac393ff1e8 azureFirstPartyApplicationClientId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX azureMiMockServicePrincipalCertName:msiMockPoolCert-6 azureMiMockServicePrincipalClientId:fe0ac67d-d1d8-45f9-a347-362201553034 azureMiMockServicePrincipalPrincipalId:f8be0ede-39df-41cf-aed7-7f3626f23a5a azureOperatorsMI:map[dev:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] public:map[cloudControllerManager:map[roleDefinitions:[map[id:a1f96423-95ce-4224-ab27-4e3dc72facd4 name:Azure Red Hat OpenShift Cloud Controller Manager]]] cloudNetworkConfig:map[roleDefinitions:[map[id:be7a6435-15ae-4171-8f30-4a343eff9e8f name:Azure Red Hat OpenShift Network Operator]]] clusterApiAzure:map[roleDefinitions:[map[id:88366f10-ed47-4cc0-9fab-c8a06148393e name:Azure Red Hat OpenShift Hosted Control Planes Cluster API Provider]]] controlPlane:map[roleDefinitions:[map[id:fc0c873f-45e9-4d0d-a7d1-585aab30c6ed name:Azure Red Hat OpenShift Hosted Control Planes Control Plane Operator]]] diskCsiDriver:map[roleDefinitions:[map[id:5b7237c5-45e1-49d6-bc18-a1f62f400748 name:Azure Red Hat OpenShift Disk Storage Operator]]] fileCsiDriver:map[roleDefinitions:[map[id:0d7aedc0-15fd-4a67-a412-efad370c947e name:Azure Red Hat OpenShift File Storage Operator]]] imageRegistry:map[roleDefinitions:[map[id:8b32b316-c2f5-4ddf-b05b-83dacd2d08b5 name:Azure Red Hat OpenShift Image Registry Operator]]] ingress:map[roleDefinitions:[map[id:0336e1d3-7a87-462b-b6db-342b63f7802c name:Azure Red Hat OpenShift Cluster Ingress Operator]]] kms:map[roleDefinitions:[map[id:12338af0-0e69-4776-bea7-57ae8d297424 name:Key Vault Crypto User]]] serviceManagedIdentity:map[roleDefinitions:[map[id:c0ff367d-66d8-445e-917c-583feb0ef0d4 name:Azure Red Hat OpenShift Hosted Control Planes Service Managed Identity]]]] roleSetName:dev] backplaneURL: batchProcesses: batchProcessesDryRun:true clientId:foo clientScopes:openid clientSecret:bar clusterErrorReport:false cosmosDatabaseName:arohcpci01-rp-j9715712 cosmosDatabaseUrl:https://arohcpci01-rp-j9715712.documents.azure.com:443/ cpuRequest:630m csDeploymentStrategy:map[rollingUpdate:map[maxSurge:1 maxUnavailable:0] type:RollingUpdate] dataPlaneHAProxyImage:arohcpocpdev.azurecr.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-shared-ingress-main@sha256:5e22710fe440869655329ecc0ef57b4ae15f4ef96e74a8904b1e6fafc0d87678 databaseAuthMethod:postgres databaseDisableTls:true databaseHost:ocm-cs-db databaseK8sSecretName:ocm-cs-db databaseName:ocm-cs-db databasePassword:TheBlurstOfTimes databasePort:5432 databaseServiceName:ocm-cs-db databaseUser:ocm databaseVolumeCapacity:512Mi debugMaxDuration:5m debugPort: defaultExpiration:0 deployment:map[zoneCount:3] environment:arohcpci01 forceMigration: fpaCertName:firstPartyCert2 gatewayUrl:http://127.0.0.1:9090 global:map[imageDigest:sha256:11c94e058bf6e19d183bb655461fb5d37d7368243382944adabc1c0fdf10ea9b] imageDigest:sha256:11c94e058bf6e19d183bb655461fb5d37d7368243382944adabc1c0fdf10ea9b imageRegistry:arohcpsvcdev.azurecr.io imageRepository:mvacula/cs insecure:false jwksUrl:http://localhost logLevel:4 machinePoolMigrationWorkerPeriod:1h managedIdentitiesDataPlaneAudienceResource:https://dummy.org maximumExpiration:0 memoryLimit:unlimited memoryRequest:150Mi ocpAcrResourceId:/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/global/providers/Microsoft.ContainerRegistry/registries/arohcpocpdev ocpAcrResourceUrl:arohcpocpdev.azurecr.io oidcIssuerBaseUrl:https://arohcpci01oidcj9715712.z1.web.core.windows.net/ oidcIssuerBlobServiceUrl:https://arohcpci01oidcj9715712.blob.core.windows.net/ postgresqlVersion:12 provisionShardClusterLimit:100 pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:mvacula/cs:pull workloadIdentityClientId:c964e23b-7c92-4e91-82c7-8ed9b8848c7e workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] region:westus3 replicas:3 runtimeMode:aro-hcp serviceAccountName:clusters-service serviceKeyVaultName:aro-hcp-dev-svc-kv tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX tlsCertificatesEnabled:true tlsCertificatesIssuer:Self tokenUrl:http://localhost useAzureDB:false userDefinedDnsBaseDomain:i1.devshift.org]\"\ntime=2026-09-29T12:22:58.550Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-09-29T12:22:58.851Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy namespace=clusters-service\ntime=2026-09-29T12:22:59.239Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy dryRun=false\ntime=2026-09-29T12:22:59.239Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-09-29T12:22:59.359Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy\ntime=2026-09-29T12:33:06.293Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"resource Deployment/clusters-service/clusters-service not ready. status: Failed, message: Progress deadline exceeded\"\ntime=2026-09-29T12:33:06.294Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: Failed, message: Progress deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c62be99dc720ba32ad3cf1cf43bbf7c29367865dd80cd9f2ab3e6ab752f3db61","failure_pattern":"failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: Failed, message: Progress deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource Deployment/clusters-service/clusters-service not ready. status: Failed, message: Progress deadline exceeded"}},{"row_id":"432c359f0debf98314f25317c83c445fb0065ee5720fbead1b6a1e9a4b8059da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104901125449715712","occurred_at":"2026-09-29T11:48:30Z","failed_at":"alert","signature_id":"1e62489e7866f0527d21798b1edc54e005bcf63e94abdc30a0b7d8c87828bc77","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T12:28:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9715712-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-54c9ff565c-wwtqq\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-54c9ff565c-wwtqq has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-29T12:28:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9715712-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-54c9ff565c-rqgr7\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-54c9ff565c-rqgr7 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-29T12:28:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9715712-svc\", component=\"kubernetes-infrastructure\", namespace=\"clusters-service\", pod=\"clusters-service-54c9ff565c-dp5lq\", region=\"westus3\", severity=\"warning\"\n Description: Pod clusters-service/clusters-service-54c9ff565c-dp5lq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","c62be99dc720ba32ad3cf1cf43bbf7c29367865dd80cd9f2ab3e6ab752f3db61"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104896461568217088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T11:29:58Z","started_at":"2026-09-29T11:29:58Z","completed_at":"2026-09-29T13:44:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T13:54:23.828628893Z","timing_metadata_checked_at":"2026-09-29T13:54:23.828628893Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T13:54:23.831499282Z","region_metadata_checked_at":"2026-09-29T13:54:23.831499282Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"15c4c3c4bb3d66f46bd8bc6c82638db7558b73d8548e59de06d9ac97def7b55e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104896461568217088","occurred_at":"2026-09-29T11:29:58Z","failed_at":"alert","signature_id":"c58c071b10807a12470cdf0b7562c01a13053e5606800d1cf8bbe023c5be6e0f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T12:31:18Z\n Ended: 2026-09-29T12:40:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8217088-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4lslm5i8q6f369o8t992bo6sevfbrh-p0z2h6h5a9y0w0j\", pod=\"router-8577fddd95-8qm6v\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4lslm5i8q6f369o8t992bo6sevfbrh-p0z2h6h5a9y0w0j/router-8577fddd95-8qm6v has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"b828dc97d5a007108b37a373d6c9b03c7d4596f3fe63a1f8c1d0aa1998dcc71b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104896461568217088","occurred_at":"2026-09-29T11:29:58Z","failed_at":"e2e","signature_id":"fcf5808b39bc5124c5c5b76942293c53c774a92cb7f926e692ed9814d28eb97f","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001152330\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001152330\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104895962802556928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"0bdd36fc9bed8d937676c8592b848cd9cbcde70d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T11:27:59Z","started_at":"2026-09-29T11:27:59Z","completed_at":"2026-09-29T13:51:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T13:54:23.445246913Z","timing_metadata_checked_at":"2026-09-29T13:54:23.445246913Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T13:54:23.448362201Z","region_metadata_checked_at":"2026-09-29T13:54:23.448362201Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7463127c470989ec55e4d035a610c35333f172fbcd515bfccfe07cd06609cc8d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104895962802556928","occurred_at":"2026-09-29T11:27:59Z","failed_at":"e2e","signature_id":"fc07e91b9dfd5eecaa0304b28428db3700d4b80ea107d05a261ab60b39464230","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00160d368\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00160d368\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104892283974324224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"closed","pr_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","final_merged_sha":"3c72ea92559bff30660a213a1ce63775f8a085c7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T11:13:21Z","started_at":"2026-09-29T11:13:21Z","completed_at":"2026-09-29T13:56:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T14:04:40.678674081Z","timing_metadata_checked_at":"2026-09-29T14:04:40.678674081Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T14:04:40.68502106Z","region_metadata_checked_at":"2026-09-29T14:04:40.68502106Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"22446bf35043a6b3bb58049e56d3ccafef849b08d6ade15cfeab9052eb0cd966","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104892283974324224","occurred_at":"2026-09-29T11:13:21Z","failed_at":"alert","signature_id":"c8fb3e5665b8c85fed2f572fa2ac02f42c7181c584190d195168fcd680376311","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T12:10:11Z\n Ended: 2026-09-29T12:36:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4324224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4ljqd98a0ltiqds61qu1uqil6ns75e-v20261001\", pod=\"router-c85f86bb6-554fp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4ljqd98a0ltiqds61qu1uqil6ns75e-v20261001/router-c85f86bb6-554fp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T12:11:12Z\n Ended: 2026-09-29T12:39:15Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4324224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4ljt7cibn81sb7upchqksdk1o3pk6k-l2c2c2w5b6r7a5t\", pod=\"router-6c8dc8d9f9-k5vdh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4ljt7cibn81sb7upchqksdk1o3pk6k-l2c2c2w5b6r7a5t/router-6c8dc8d9f9-k5vdh has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T12:12:13Z\n Ended: 2026-09-29T12:43:12Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4324224-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4ll36tvrrrt72ee2s4e8n92bvd8a7l-agg-cluster\", pod=\"router-5c8466799d-l8fcc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4ll36tvrrrt72ee2s4e8n92bvd8a7l-agg-cluster/router-5c8466799d-l8fcc has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"497d89b778d7b7d7f8091b022a0e95f80894aa232c52aae5bd3163ae1b6b60b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104892283974324224","occurred_at":"2026-09-29T11:13:21Z","failed_at":"e2e","signature_id":"7cda01b35a281494f1568ad8ae621f568043b431e5e95325c2cb32fa07466532","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-n7b4hv6c4kdc/cluster-candidate-4-21-ctstkf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000684460\u003e: \n failed to create HCP cluster cluster-candidate-4-21-ctstkf: failed waiting for cluster=\"cluster-candidate-4-21-ctstkf\" in resourcegroup=\"rg-candidate-4-21-n7b4hv6c4kdc\" to finish creating: GET https://rp.j4324224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90eb9bd3-e545-4675-92d4-1a9db7addf78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90eb9bd3-e545-4675-92d4-1a9db7addf78\",\n \"name\": \"90eb9bd3-e545-4675-92d4-1a9db7addf78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T11:58:43.089165396Z\",\n \"endTime\": \"2026-09-29T12:27:51.615294584Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m5s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-n7b4hv6c4kdc/cluster-candidate-4-21-ctstkf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000684460\u003e: \n failed to create HCP cluster cluster-candidate-4-21-ctstkf: failed waiting for cluster=\"cluster-candidate-4-21-ctstkf\" in resourcegroup=\"rg-candidate-4-21-n7b4hv6c4kdc\" to finish creating: GET https://rp.j4324224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90eb9bd3-e545-4675-92d4-1a9db7addf78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/90eb9bd3-e545-4675-92d4-1a9db7addf78\",\n \"name\": \"90eb9bd3-e545-4675-92d4-1a9db7addf78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T11:58:43.089165396Z\",\n \"endTime\": \"2026-09-29T12:27:51.615294584Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m5s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4ac98aa618282b8e659bcd9e3c707d99d3b1420386855cb37c8337e6c55c9e6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104892283974324224","occurred_at":"2026-09-29T11:13:21Z","failed_at":"e2e","signature_id":"1020fd15b746938ab2267227fb82d98cdfd040508cfced2fa835b50816b68943","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f85160\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-pgsj5htdjqmn\" to finish creating: GET https://rp.j4324224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34ec8b87-ca47-4120-ac5d-75d27028bbb0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34ec8b87-ca47-4120-ac5d-75d27028bbb0\",\n \"name\": \"34ec8b87-ca47-4120-ac5d-75d27028bbb0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T12:01:16.114821395Z\",\n \"endTime\": \"2026-09-29T12:30:21.231106284Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m45s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f85160\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-pgsj5htdjqmn\" to finish creating: GET https://rp.j4324224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34ec8b87-ca47-4120-ac5d-75d27028bbb0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/34ec8b87-ca47-4120-ac5d-75d27028bbb0\",\n \"name\": \"34ec8b87-ca47-4120-ac5d-75d27028bbb0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T12:01:16.114821395Z\",\n \"endTime\": \"2026-09-29T12:30:21.231106284Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m45s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9423db7a9fea6786843135de9725179850d0fc2e51a550083a324c2d94be5765","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104892283974324224","occurred_at":"2026-09-29T11:13:21Z","failed_at":"e2e","signature_id":"8326aa2b1b28620268b45af3209475a037d87152e0d5261f424e37bb8d32afc3","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004d8a40\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-s7xdrhrsgd89\" to finish creating: GET https://rp.j4324224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6a4dda3b-cac2-403f-a6f9-4260cafb4d11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6a4dda3b-cac2-403f-a6f9-4260cafb4d11\",\n \"name\": \"6a4dda3b-cac2-403f-a6f9-4260cafb4d11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T11:58:32.024627585Z\",\n \"endTime\": \"2026-09-29T12:27:41.678607113Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004d8a40\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-s7xdrhrsgd89\" to finish creating: GET https://rp.j4324224.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6a4dda3b-cac2-403f-a6f9-4260cafb4d11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6a4dda3b-cac2-403f-a6f9-4260cafb4d11\",\n \"name\": \"6a4dda3b-cac2-403f-a6f9-4260cafb4d11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T11:58:32.024627585Z\",\n \"endTime\": \"2026-09-29T12:27:41.678607113Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9ce9a083d16b9c16b5c90319c6ff4be8b9d44b90c048e30f90d5f6902aabb5dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104892283974324224","occurred_at":"2026-09-29T11:13:21Z","failed_at":"e2e","signature_id":"05f50fe6c06a19d52aaf0828883deb3ec0137078e1331dc21248328defadaed8","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00090ce00\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-26c6r9w89qkm\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00090ce00\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-26c6r9w89qkm\" to finish creating: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-26c6r9w89qkm\" to finish creating: context deadline exceeded"}},{"row_id":"b008ef4dc8633471ddad69740de2e005b126681c76e8aa704bd9d5b992997fc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104892283974324224","occurred_at":"2026-09-29T11:13:21Z","failed_at":"e2e","signature_id":"b0b9c099217ebc7857ac276feb0f9f9618cf47ea458e2f83dc0f79b2a461ccbf","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0002b4fa8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0002b4fa8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"d0c216b114e08679584444122011ad2b4f9ea16ea66321188d6f11dd97b29691","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104892283974324224","occurred_at":"2026-09-29T11:13:21Z","failed_at":"alert","signature_id":"8a5b2af1a0b796c047e659da6a5145d49b003301aa4637e08b26287323dd087f","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T11:58:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4324224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-qkzmp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-qkzmp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-29T11:58:29Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4324224-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-5685964bfc-wkt7w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod monitoring/kube-events-5685964bfc-wkt7w has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"d660759d1a6164f564b07e1105634b05a73349ac134bb78aeb0b657b2b09c9fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104892283974324224","occurred_at":"2026-09-29T11:13:21Z","failed_at":"alert","signature_id":"7d7380d9fd646c23c9e7705bd8f538a45f322bc1768ade95d561c5e5baa251ff","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T13:36:58Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4324224-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.28:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4324224/providers/microsoft.monitor/accounts/services-j4324224\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5cbcfdd97b-8b5k4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4324224-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-29T13:38:02Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j4324224-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.28:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4324224/providers/microsoft.monitor/accounts/services-j4324224\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-5cbcfdd97b-8b5k4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j4324224-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104878392305258496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"open","pr_sha":"2a88f4ef769f26f243ea6c58499709d5313caf23","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T10:18:10Z","started_at":"2026-09-29T10:18:10Z","completed_at":"2026-09-29T12:34:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T12:50:34.305809677Z","timing_metadata_checked_at":"2026-09-29T12:50:34.305809677Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T12:50:34.308787471Z","region_metadata_checked_at":"2026-09-29T12:50:34.308787471Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"093e6181e0ff37c246e2cd47473b1e1c90901abec5b64fa9ada1062b3d310bb7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104878392305258496","occurred_at":"2026-09-29T10:18:10Z","failed_at":"e2e","signature_id":"511539107c7c03b414c5b1504c63e6de9fec5b16cdfa335fe9168472c15fe7ec","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0010197b8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0010197b8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7239/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104872117722943488","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7239,"pr_state":"open","pr_sha":"ed0294e0dd93848b565f350eb0bc653655163ca8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T09:53:14Z","started_at":"2026-09-29T09:53:14Z","completed_at":"2026-09-29T12:13:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T12:29:32.349242173Z","timing_metadata_checked_at":"2026-09-29T12:29:32.349242173Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T12:29:32.351784264Z","region_metadata_checked_at":"2026-09-29T12:29:32.351784264Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"7ed30a02b9e1b83c6821722f24a095e30aaa03c694458c47d2b90ef55665ae18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7239/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104872117722943488","occurred_at":"2026-09-29T09:53:14Z","failed_at":"e2e","signature_id":"1a917fc830a0d12716eab0604924d59f3df6d4e6b5e48f546624a92a33952d60","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.005s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0019ab848\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.005s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0019ab848\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"b84500068939707a57eb7fc12aed4e1eaf4f9c3aec2450a46badc68fe78f5088","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7239/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104872117722943488","occurred_at":"2026-09-29T09:53:14Z","failed_at":"e2e","signature_id":"da5a3897ba2157f097ec0508d7ff9fe82cf45ed4377c8d9830f6e3a5a8afbe00","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9c100\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-mmcdpztn5v9t\" to finish creating: GET https://rp.j2943488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0ac3d63-a59c-461d-8dfa-d9b2e95881c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0ac3d63-a59c-461d-8dfa-d9b2e95881c7\",\n \"name\": \"d0ac3d63-a59c-461d-8dfa-d9b2e95881c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:57:27.381307042Z\",\n \"endTime\": \"2026-09-29T11:16:29.731763604Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g9b6e9g2n8g2k2z-np-1-zgpwl-bztss: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-mmcdpztn5v9t--managed/providers/Microsoft.Compute/virtualMachines/g9b6e9g2n8g2k2z-np-1-zgpwl-bztss to exist\\nMachine g9b6e9g2n8g2k2z-np-1-zgpwl-ttrd9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-mmcdpztn5v9t--managed/providers/Microsoft.Compute/virtualMachines/g9b6e9g2n8g2k2z-np-1-zgpwl-ttrd9 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g9b6e9g2n8g2k2z-np-1-zgpwl-bztss: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-mmcdpztn5v9t--managed/providers/Microsoft.Compute/virtualMachines/g9b6e9g2n8g2k2z-np-1-zgpwl-bztss to exist\\nMachine g9b6e9g2n8g2k2z-np-1-zgpwl-ttrd9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-mmcdpztn5v9t--managed/providers/Microsoft.Compute/virtualMachines/g9b6e9g2n8g2k2z-np-1-zgpwl-ttrd9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9c100\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-mmcdpztn5v9t\" to finish creating: GET https://rp.j2943488.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0ac3d63-a59c-461d-8dfa-d9b2e95881c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d0ac3d63-a59c-461d-8dfa-d9b2e95881c7\",\n \"name\": \"d0ac3d63-a59c-461d-8dfa-d9b2e95881c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:57:27.381307042Z\",\n \"endTime\": \"2026-09-29T11:16:29.731763604Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine g9b6e9g2n8g2k2z-np-1-zgpwl-bztss: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-mmcdpztn5v9t--managed/providers/Microsoft.Compute/virtualMachines/g9b6e9g2n8g2k2z-np-1-zgpwl-bztss to exist\\nMachine g9b6e9g2n8g2k2z-np-1-zgpwl-ttrd9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-mmcdpztn5v9t--managed/providers/Microsoft.Compute/virtualMachines/g9b6e9g2n8g2k2z-np-1-zgpwl-ttrd9 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine g9b6e9g2n8g2k2z-np-1-zgpwl-bztss: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-mmcdpztn5v9t--managed/providers/Microsoft.Compute/virtualMachines/g9b6e9g2n8g2k2z-np-1-zgpwl-bztss to exist\\nMachine g9b6e9g2n8g2k2z-np-1-zgpwl-ttrd9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-mmcdpztn5v9t--managed/providers/Microsoft.Compute/virtualMachines/g9b6e9g2n8g2k2z-np-1-zgpwl-ttrd9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871453622013952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"e7500a3658c1692b6c32705cb7edf403b704aae4","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T09:50:35Z","started_at":"2026-09-29T09:50:35Z","completed_at":"2026-09-29T12:19:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T12:29:32.133406763Z","timing_metadata_checked_at":"2026-09-29T12:29:32.133406763Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T12:29:32.139130742Z","region_metadata_checked_at":"2026-09-29T12:29:32.139130742Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3b8b40dedf32b217b369263743886d1f6d2795f2b753328ca9cc9b8af23d1752","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871453622013952","occurred_at":"2026-09-29T09:50:35Z","failed_at":"e2e","signature_id":"18309e66f8cb4168bdc21372bfe4143caa5f3a0da4946d444d3afeeb2d3d02c2","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001684ff0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001684ff0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871291713490944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T09:49:57Z","started_at":"2026-09-29T09:49:57Z","completed_at":"2026-09-29T12:22:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T12:40:02.512829854Z","timing_metadata_checked_at":"2026-09-29T12:40:02.512829854Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T12:40:02.516292942Z","region_metadata_checked_at":"2026-09-29T12:40:02.516292942Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"23025c93bda95ae60ea368eb3df20241d3222e302a2b2a45054180cdbd13adfe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871291713490944","occurred_at":"2026-09-29T09:49:57Z","failed_at":"e2e","signature_id":"d0277abb6518b0b06ec197cc4b60019e4ed65ed27d32a7e7d49556ef794bb452","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f21878\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f21878\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871248742846464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7221,"pr_state":"closed","pr_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","final_merged_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T09:49:46Z","started_at":"2026-09-29T09:49:46Z","completed_at":"2026-09-29T12:24:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T12:40:02.633196138Z","timing_metadata_checked_at":"2026-09-29T12:40:02.633196138Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T12:40:02.635886329Z","region_metadata_checked_at":"2026-09-29T12:40:02.635886329Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"a51719765fa3c9ded1b0ac5f8ed7d60f9475310c8a39fd38fdbf85f062b3c5cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871248742846464","occurred_at":"2026-09-29T09:49:46Z","failed_at":"e2e","signature_id":"6b418d69171bd9da7843dfede67cb76e1ef5fdb066024cc8e7447f8056700f80","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f13890\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f13890\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"c63b155b2cdc58861a3403c6d3dcd5988afbcd3c09f53fb070a99bff999bc7f3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871248742846464","occurred_at":"2026-09-29T09:49:46Z","failed_at":"alert","signature_id":"40257ee779591b1832dfd2cdbdc95bef569afcfe7764c2215c7427baad158e9e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 9 time(s) (3 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T11:05:59Z\n Ended: 2026-09-29T11:17:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2846464-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4km35ikrnfnt459qng9pc4av7qb8aa-o4c9e9c1n2y2v7k\", pod=\"router-548c9b5544-27k5t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4km35ikrnfnt459qng9pc4av7qb8aa-o4c9e9c1n2y2v7k/router-548c9b5544-27k5t has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T11:07:57Z\n Ended: 2026-09-29T11:17:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2846464-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4kn98kq4p2f926sh42782e09tonqrr-agg-cluster\", pod=\"router-6fcb4bd844-cvdrt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4kn98kq4p2f926sh42782e09tonqrr-agg-cluster/router-6fcb4bd844-cvdrt has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T11:07:57Z\n Ended: 2026-09-29T11:17:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2846464-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4kmvj8c1o1379gv6jm7ftg9jm6r9g0-cilium-cluster\", pod=\"router-55d44c5959-f8qmj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4kmvj8c1o1379gv6jm7ftg9jm6r9g0-cilium-cluster/router-55d44c5959-f8qmj has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871228572438528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7224,"pr_state":"closed","pr_sha":"a6cabcda34fdee6066dd2b496192fcb45072451a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T09:49:42Z","started_at":"2026-09-29T09:49:42Z","completed_at":"2026-09-29T12:15:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T12:29:32.15605078Z","timing_metadata_checked_at":"2026-09-29T12:29:32.15605078Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T12:29:32.15883777Z","region_metadata_checked_at":"2026-09-29T12:29:32.15883777Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3aa1b35d5817f60ee68c0248a0bf56751edeccb1b4edb3f62cc5e41c4409caba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871228572438528","occurred_at":"2026-09-29T09:49:42Z","failed_at":"e2e","signature_id":"9a10456e5e9d1c6aa0d5c183b7eda2696959c43695ac4d4eb73736f087028690","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013503d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0013503d8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871211610673152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T09:49:38Z","started_at":"2026-09-29T09:49:38Z","completed_at":"2026-09-29T12:22:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T12:40:02.613538706Z","timing_metadata_checked_at":"2026-09-29T12:40:02.613538706Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T12:40:02.616158397Z","region_metadata_checked_at":"2026-09-29T12:40:02.616158397Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"c667f164509318b2315349ea525912772bcabfa552f829cbf293e27b30d9538d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871211610673152","occurred_at":"2026-09-29T09:49:38Z","failed_at":"e2e","signature_id":"073eef8f1e085922d2edab4cfed7339689524abbb91d122efe48e902887373e6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d6tfd4\" for cluster \"cluster-candidate-4-23-d6tfd4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00015e780\u003e: \n failed to create NodePool np-d6tfd4: failed waiting for nodepool=\"np-d6tfd4\" for cluster \"cluster-candidate-4-23-d6tfd4\" in resourcegroup=\"rg-candidate-4-23-4bxj27pc7glv\" to finish creating: GET https://rp.j0673152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/890f473a-a40f-448f-962e-d96b38ea1a2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/890f473a-a40f-448f-962e-d96b38ea1a2b\",\n \"name\": \"890f473a-a40f-448f-962e-d96b38ea1a2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T11:06:17.349879973Z\",\n \"endTime\": \"2026-09-29T11:25:25.480122641Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-9xdl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4bxj27pc7glv-candidate-d6tfd4--managed/providers/Microsoft.Compute/virtualMachines/l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-9xdl5 to exist\\nMachine l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-z2hws: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4bxj27pc7glv-candidate-d6tfd4--managed/providers/Microsoft.Compute/virtualMachines/l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-z2hws to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-9xdl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4bxj27pc7glv-candidate-d6tfd4--managed/providers/Microsoft.Compute/virtualMachines/l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-9xdl5 to exist\\nMachine l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-z2hws: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4bxj27pc7glv-candidate-d6tfd4--managed/providers/Microsoft.Compute/virtualMachines/l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-z2hws to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d6tfd4\" for cluster \"cluster-candidate-4-23-d6tfd4\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00015e780\u003e: \n failed to create NodePool np-d6tfd4: failed waiting for nodepool=\"np-d6tfd4\" for cluster \"cluster-candidate-4-23-d6tfd4\" in resourcegroup=\"rg-candidate-4-23-4bxj27pc7glv\" to finish creating: GET https://rp.j0673152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/890f473a-a40f-448f-962e-d96b38ea1a2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/890f473a-a40f-448f-962e-d96b38ea1a2b\",\n \"name\": \"890f473a-a40f-448f-962e-d96b38ea1a2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T11:06:17.349879973Z\",\n \"endTime\": \"2026-09-29T11:25:25.480122641Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-9xdl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4bxj27pc7glv-candidate-d6tfd4--managed/providers/Microsoft.Compute/virtualMachines/l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-9xdl5 to exist\\nMachine l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-z2hws: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4bxj27pc7glv-candidate-d6tfd4--managed/providers/Microsoft.Compute/virtualMachines/l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-z2hws to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-9xdl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4bxj27pc7glv-candidate-d6tfd4--managed/providers/Microsoft.Compute/virtualMachines/l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-9xdl5 to exist\\nMachine l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-z2hws: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-4bxj27pc7glv-candidate-d6tfd4--managed/providers/Microsoft.Compute/virtualMachines/l2q5s1i5v0u1a0f-np-d6tfd4-vnx94-z2hws to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cabb6985b0ee88c407d660be61eae1fb19851f5638c78d2bad9585840035c91f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104871211610673152","occurred_at":"2026-09-29T09:49:38Z","failed_at":"e2e","signature_id":"8198c353c7d3001ebae184767e84718503000d2c358aae239e9b18ff430c32b7","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000fb1230\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000fb1230\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104866139002638336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7165,"pr_state":"open","pr_sha":"4ada4bfca590fd84b0b6be17befd2127f1df09bf","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T09:29:28Z","started_at":"2026-09-29T09:29:28Z","completed_at":"2026-09-29T11:42:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T11:58:02.74880628Z","timing_metadata_checked_at":"2026-09-29T11:58:02.74880628Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T11:58:02.753029569Z","region_metadata_checked_at":"2026-09-29T11:58:02.753029569Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0b0d18b877a1be03b9aa25dd0975c6795dea7d9622c5854c9de490985755b3dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104866139002638336","occurred_at":"2026-09-29T09:29:28Z","failed_at":"e2e","signature_id":"d1d969a7756e4e950b0981cf4307e261652ff463e2de932a38407988a79b62ad","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-mswjhw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017814e0\u003e: \n DNS for route host agnhost-e2e-sample-app-qg2f7.apps.aro.t6o0l8c1g4g9c6n.o5er.j2638336.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-qg2f7.apps.aro.t6o0l8c1g4g9c6n.o5er.j2638336.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-qg2f7.apps.aro.t6o0l8c1g4g9c6n.o5er.j2638336.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-mswjhw\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017814e0\u003e: \n DNS for route host agnhost-e2e-sample-app-qg2f7.apps.aro.t6o0l8c1g4g9c6n.o5er.j2638336.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-qg2f7.apps.aro.t6o0l8c1g4g9c6n.o5er.j2638336.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-qg2f7.apps.aro.t6o0l8c1g4g9c6n.o5er.j2638336.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-qg2f7.apps.aro.t6o0l8c1g4g9c6n.o5er.j2638336.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-qg2f7.apps.aro.t6o0l8c1g4g9c6n.o5er.j2638336.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-qg2f7.apps.aro.t6o0l8c1g4g9c6n.o5er.j2638336.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"1f3fd497a20a4e66e7c99a934b7c85294454a4167d81c8f1e2c0ddbaebc014b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7165/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104866139002638336","occurred_at":"2026-09-29T09:29:28Z","failed_at":"e2e","signature_id":"b6b4ebbd65327f891ddf6a4f61886643ec7ea1eeac6b0f1a4d20c00cb544de75","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001516438\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001516438\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104861480800227328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6944,"pr_state":"open","pr_sha":"662dc5197cfccd3edc3edff175ee06ad7a73a528","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T09:10:58Z","started_at":"2026-09-29T09:10:58Z","completed_at":"2026-09-29T09:47:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:53:06.220110846Z","timing_metadata_checked_at":"2026-09-29T09:53:06.220110846Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:53:06.227576922Z","region_metadata_checked_at":"2026-09-29T09:53:06.227576922Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e59372925bb03bd9c9e2e8493f801bca6f6aa5a978cc0cf5dd9ab5f5bfbca821","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6944/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104861480800227328","occurred_at":"2026-09-29T09:10:58Z","failed_at":"provision","signature_id":"116580a5983478d30cd90ef051e618ba8304f4b201ee8925ed5485a5fb502283","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-29T09:32:36.709Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-29T09:40:14.775Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-29T09:32:36.720660292Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-29T09:32:38.364028078Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-29T09:32:38.559062049Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T09:40:14.773569619Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"939DC55C48692422\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0227328-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j0227328-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T09:38:15.5365612Z\\\",\\\"duration\\\":\\\"PT1M53.6748858S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T09:40:14.773629341Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"7244CB440D997254\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0227328-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j0227328-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T09:38:15.5365612Z\\\",\\\"duration\\\":\\\"PT1M23.3840019S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T09:40:14.773642111Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1CB00462E228B7F2\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0227328-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j0227328-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-29T09:38:15.5365612Z\\\",\\\"duration\\\":\\\"PT1M22.8983991S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-52905658-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-52905658-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-29T09:40:14.773711183Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2656F8058F363717\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0227328-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j0227328-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T09:38:15.5365612Z\\\",\\\"duration\\\":\\\"PT1M53.6953657S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T09:40:14.773723573Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"14B4CF500D615D35\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0227328-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j0227328-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T09:32:43.8316068Z\\\",\\\"duration\\\":\\\"PT6M54.993084S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T09:40:14.773734363Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"389A184917075DF4\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0227328-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T09:32:43.8316068Z\\\",\\\"duration\\\":\\\"PT5M31.5756796S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T09:40:14.773756484Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0227328-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0227328-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0227328-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109325266766857\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0227328-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-52905658-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0227328-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-52905658-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m38.053105252s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104861197303025664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"0bdd36fc9bed8d937676c8592b848cd9cbcde70d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T09:09:50Z","started_at":"2026-09-29T09:09:50Z","completed_at":"2026-09-29T11:17:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T11:36:48.569451529Z","timing_metadata_checked_at":"2026-09-29T11:36:48.569451529Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T11:36:48.573542515Z","region_metadata_checked_at":"2026-09-29T11:36:48.573542515Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"07d70806125b5848848678c8d56c36e1fb3756569e8503849d5895694cbad977","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104861197303025664","occurred_at":"2026-09-29T09:09:50Z","failed_at":"e2e","signature_id":"d9369573931e6f4cbd6449da296e2051200f2351aa6cd8b5fb8a6886570f5cc2","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001221b00\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc001221b00\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"5d1c2353e826fabe343df6828ed836d4e561972f53c014e1c5649b247f3ce830","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104861197303025664","occurred_at":"2026-09-29T09:09:50Z","failed_at":"alert","signature_id":"fb7cbb7a29d32f4b826f3d9f6551e5199b49793fe6f5ef3beda6da61fc7e9133","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T10:27:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-29T10:27:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T10:33:53Z\n Ended: 2026-09-29T10:49:56Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4jo44vp2jv0nlkcl93iohtjucb2f9b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment klusterlet-2t4jo44vp2jv0nlkcl93iohtjucb2f9b/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-29T10:33:53Z\n Ended: 2026-09-29T10:49:56Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4jo44vp2jv0nlkcl93iohtjucb2f9b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment klusterlet-2t4jo44vp2jv0nlkcl93iohtjucb2f9b/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"9857b9502eec89934679eef1503aae73795ce2992d25b7bbd12f76842782d39d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104861197303025664","occurred_at":"2026-09-29T09:09:50Z","failed_at":"alert","signature_id":"d747cb335f976610cd68c7f3a5d4d4654942b38d56c89998ee0fc8d459302a24","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T09:52:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-78648d9c8d-q46qb\", region=\"centralus\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-78648d9c8d-q46qb has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"ef03f84a93821e1e389fa749e30335239f3050ea9d17ac00c5c7f0b96e261ef3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104861197303025664","occurred_at":"2026-09-29T09:09:50Z","failed_at":"alert","signature_id":"a87232fe42fd99e0f727dc4f690fd9d2ffd6da39cc86cf60bc45e371c81d79e4","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 14 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T10:17:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-29T10:17:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T10:22:52Z\n Ended: 2026-09-29T10:32:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4jnvs8kd2mj67qj3qjdr0prgmk3g16\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4jnvs8kd2mj67qj3qjdr0prgmk3g16/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-29T10:22:52Z\n Ended: 2026-09-29T10:32:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4jnvs8kd2mj67qj3qjdr0prgmk3g16\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4jnvs8kd2mj67qj3qjdr0prgmk3g16/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-29T10:23:53Z\n Ended: 2026-09-29T10:49:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4jo44vp2jv0nlkcl93iohtjucb2f9b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4jo44vp2jv0nlkcl93iohtjucb2f9b/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-29T10:23:53Z\n Ended: 2026-09-29T10:49:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4jo44vp2jv0nlkcl93iohtjucb2f9b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4jo44vp2jv0nlkcl93iohtjucb2f9b/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-29T10:41:53Z\n Ended: 2026-09-29T10:53:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4k0pk1qfbh6ckcvag872v8bpqll942\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4k0pk1qfbh6ckcvag872v8bpqll942/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-29T10:41:53Z\n Ended: 2026-09-29T10:50:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4k0nh3n7qatpi8mc239v6qipvko07d\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4k0nh3n7qatpi8mc239v6qipvko07d/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-29T10:41:53Z\n Ended: 2026-09-29T10:55:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4k0pk1qfbh6ckcvag872v8bpqll942\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4k0pk1qfbh6ckcvag872v8bpqll942/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-29T10:41:53Z\n Ended: 2026-09-29T10:50:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4k0nh3n7qatpi8mc239v6qipvko07d\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4k0nh3n7qatpi8mc239v6qipvko07d/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-29T10:45:53Z\n Ended: 2026-09-29T10:55:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4k2egjlp01qontmh09pkq2jcahc57j\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4k2egjlp01qontmh09pkq2jcahc57j/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-29T10:45:53Z\n Ended: 2026-09-29T10:55:00Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4k2egjlp01qontmh09pkq2jcahc57j\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4k2egjlp01qontmh09pkq2jcahc57j/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-29T10:46:53Z\n Ended: 2026-09-29T10:57:58Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4k3g6c5jmg9c3neja6l4kgrl2qckj3\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4k3g6c5jmg9c3neja6l4kgrl2qckj3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-29T10:46:53Z\n Ended: 2026-09-29T10:57:58Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3025664-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.135:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3025664/providers/microsoft.monitor/accounts/services-j3025664\", namespace=\"klusterlet-2t4k3g6c5jmg9c3neja6l4kgrl2qckj3\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-gxfn6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t4k3g6c5jmg9c3neja6l4kgrl2qckj3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7073,"pr_state":"open","pr_sha":"b9d120f5fe68fda8bac75fac89516e32b56c6e6e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T09:05:08Z","started_at":"2026-09-29T09:05:08Z","completed_at":"2026-09-29T11:08:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T11:26:35.500524901Z","timing_metadata_checked_at":"2026-09-29T11:26:35.500524901Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T11:26:35.504390387Z","region_metadata_checked_at":"2026-09-29T11:26:35.504390387Z"},"failed_at":["alert","e2e"],"failed_test_count":67,"occurrences":[{"row_id":"02d4da3a75642ae2935adec9517e39934ec9dc4fa1ed78c40febd3656e9dc76b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"b0ac1dabcf383a907d1be822548c593af5a702f29f020b43ba73d677baaeb233","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.1","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-pk6bpv6n88t6/cluster-candidate-5-1-7n7srh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c08c0\u003e: \n failed to create HCP cluster cluster-candidate-5-1-7n7srh: failed waiting for cluster=\"cluster-candidate-5-1-7n7srh\" in resourcegroup=\"rg-candidate-5-1-pk6bpv6n88t6\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b67c102f-2a54-49d6-8904-886e09265912\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b67c102f-2a54-49d6-8904-886e09265912\",\n \"name\": \"b67c102f-2a54-49d6-8904-886e09265912\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:22.124198854Z\",\n \"endTime\": \"2026-09-29T10:14:11.576575491Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-pk6bpv6n88t6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-7n7srh\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-pk6bpv6n88t6/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-7n7srh\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-pk6bpv6n88t6/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-7n7srh/subnets/customer-vnet-subnet-candidate-7n7srh\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-1-pk6bpv6n88t6/cluster-candidate-5-1-7n7srh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c08c0\u003e: \n failed to create HCP cluster cluster-candidate-5-1-7n7srh: failed waiting for cluster=\"cluster-candidate-5-1-7n7srh\" in resourcegroup=\"rg-candidate-5-1-pk6bpv6n88t6\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b67c102f-2a54-49d6-8904-886e09265912\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b67c102f-2a54-49d6-8904-886e09265912\",\n \"name\": \"b67c102f-2a54-49d6-8904-886e09265912\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:22.124198854Z\",\n \"endTime\": \"2026-09-29T10:14:11.576575491Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-pk6bpv6n88t6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-7n7srh\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-pk6bpv6n88t6/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-7n7srh\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-1-pk6bpv6n88t6/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-7n7srh/subnets/customer-vnet-subnet-candidate-7n7srh\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-36/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"08d6e6a5a9adac81abe8f343a9af691548d42b2c07fc9c55b31afb2c505b5aaa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"b9af01ddc08d8567ec7b105e36dba5295c588cc4cee68cbc98a0183c221433f9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-d4b6qp6rz6pg/cluster-candidate-4-22-qtgl4x should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb6ba0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-qtgl4x: failed waiting for cluster=\"cluster-candidate-4-22-qtgl4x\" in resourcegroup=\"rg-candidate-4-22-d4b6qp6rz6pg\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0880f14-6c39-4a3b-83a5-120a497ee56b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0880f14-6c39-4a3b-83a5-120a497ee56b\",\n \"name\": \"b0880f14-6c39-4a3b-83a5-120a497ee56b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:12.97924997Z\",\n \"endTime\": \"2026-09-29T10:14:22.04223532Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d4b6qp6rz6pg/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-qtgl4x\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d4b6qp6rz6pg/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-qtgl4x\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d4b6qp6rz6pg/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-qtgl4x/subnets/customer-vnet-subnet-candidate-qtgl4x\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-d4b6qp6rz6pg/cluster-candidate-4-22-qtgl4x should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb6ba0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-qtgl4x: failed waiting for cluster=\"cluster-candidate-4-22-qtgl4x\" in resourcegroup=\"rg-candidate-4-22-d4b6qp6rz6pg\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0880f14-6c39-4a3b-83a5-120a497ee56b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0880f14-6c39-4a3b-83a5-120a497ee56b\",\n \"name\": \"b0880f14-6c39-4a3b-83a5-120a497ee56b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:12.97924997Z\",\n \"endTime\": \"2026-09-29T10:14:22.04223532Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d4b6qp6rz6pg/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-qtgl4x\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d4b6qp6rz6pg/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-qtgl4x\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-d4b6qp6rz6pg/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-qtgl4x/subnets/customer-vnet-subnet-candidate-qtgl4x\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"09ab8c2ee9a98a8b71ac037b64a0ca3140a588971b14b23e936bf14e98e77d4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"b0108db75b57310bbc64dd6d362b266e89667c23f1f4ffbe0bf545375b9883f6","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-m54zlz with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1a2c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-m54zlz: failed waiting for cluster=\"np-version-upgrade-cluster-m54zlz\" in resourcegroup=\"rg-np-version-upgrade-m54zlz-t2jzcq2kmtxx\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48da7243-7838-40b3-b9b2-b29b12a5639e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48da7243-7838-40b3-b9b2-b29b12a5639e\",\n \"name\": \"48da7243-7838-40b3-b9b2-b29b12a5639e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:26:26.215372566Z\",\n \"endTime\": \"2026-09-29T10:37:12.812006886Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-m54zlz-t2jzcq2kmtxx/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-m54zlz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-m54zlz-t2jzcq2kmtxx/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-m54zlz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-m54zlz-t2jzcq2kmtxx/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-m54zlz/subnets/customer-vnet-subnet-np-upgrade-m54zlz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-m54zlz with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f1a2c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-m54zlz: failed waiting for cluster=\"np-version-upgrade-cluster-m54zlz\" in resourcegroup=\"rg-np-version-upgrade-m54zlz-t2jzcq2kmtxx\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48da7243-7838-40b3-b9b2-b29b12a5639e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/48da7243-7838-40b3-b9b2-b29b12a5639e\",\n \"name\": \"48da7243-7838-40b3-b9b2-b29b12a5639e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:26:26.215372566Z\",\n \"endTime\": \"2026-09-29T10:37:12.812006886Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-m54zlz-t2jzcq2kmtxx/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-m54zlz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-m54zlz-t2jzcq2kmtxx/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-m54zlz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-m54zlz-t2jzcq2kmtxx/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-m54zlz/subnets/customer-vnet-subnet-np-upgrade-m54zlz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"0a6fb3f486bde36747f785fb258bc688e65c9ffac998200055afd6db6947d96a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"54f527d5086a3099a6d9a9a0a91d8dcb945d22effbeca2b50ad91fc556a324a2","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000828200\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7fc2q79cg5j6\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/076f251a-bb1d-43e0-be53-ae681729ce71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/076f251a-bb1d-43e0-be53-ae681729ce71\",\n \"name\": \"076f251a-bb1d-43e0-be53-ae681729ce71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.257843137Z\",\n \"endTime\": \"2026-09-29T10:17:21.085763689Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7fc2q79cg5j6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-09/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7fc2q79cg5j6/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-09/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7fc2q79cg5j6/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-09/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000828200\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-7fc2q79cg5j6\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/076f251a-bb1d-43e0-be53-ae681729ce71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/076f251a-bb1d-43e0-be53-ae681729ce71\",\n \"name\": \"076f251a-bb1d-43e0-be53-ae681729ce71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.257843137Z\",\n \"endTime\": \"2026-09-29T10:17:21.085763689Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7fc2q79cg5j6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-09/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7fc2q79cg5j6/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-09/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7fc2q79cg5j6/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-09/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"0aa64fa91482cd178404daea593f9c6c416ca892ed868c8a6e8c059e60dd8785","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"0e3d578ed7623c4137beaf88bdb1ed7c3e426ee39eeb7a666a56aa32e3ff03f9","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc6880\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8m8djhtl6bpt\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0800f6e8-1249-44ba-abb2-835f3b99b42f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0800f6e8-1249-44ba-abb2-835f3b99b42f\",\n \"name\": \"0800f6e8-1249-44ba-abb2-835f3b99b42f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:02:19.311199081Z\",\n \"endTime\": \"2026-09-29T10:16:11.352915956Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8m8djhtl6bpt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8m8djhtl6bpt/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8m8djhtl6bpt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc6880\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8m8djhtl6bpt\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0800f6e8-1249-44ba-abb2-835f3b99b42f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0800f6e8-1249-44ba-abb2-835f3b99b42f\",\n \"name\": \"0800f6e8-1249-44ba-abb2-835f3b99b42f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:02:19.311199081Z\",\n \"endTime\": \"2026-09-29T10:16:11.352915956Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8m8djhtl6bpt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8m8djhtl6bpt/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-8m8djhtl6bpt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-08/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"12be1a6bdecc7f8611696770162c164f869143650687e0bd387587e59ac8ca4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"1c4e9f2d0960d3bb31509ff0bcacbf4060e05da057943cfda97cdc735e5ab875","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089d700\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-czwwgnnbwk5v\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e10933a-0e8d-454e-b3dd-90aa4763fa58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e10933a-0e8d-454e-b3dd-90aa4763fa58\",\n \"name\": \"1e10933a-0e8d-454e-b3dd-90aa4763fa58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:36.769605393Z\",\n \"endTime\": \"2026-09-29T10:12:41.083033268Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-czwwgnnbwk5v/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-czwwgnnbwk5v/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-czwwgnnbwk5v/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name/subnets/engineering-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089d700\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-czwwgnnbwk5v\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e10933a-0e8d-454e-b3dd-90aa4763fa58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e10933a-0e8d-454e-b3dd-90aa4763fa58\",\n \"name\": \"1e10933a-0e8d-454e-b3dd-90aa4763fa58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:36.769605393Z\",\n \"endTime\": \"2026-09-29T10:12:41.083033268Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-czwwgnnbwk5v/providers/Microsoft.Network/networkSecurityGroups/engineering-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-czwwgnnbwk5v/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-czwwgnnbwk5v/providers/Microsoft.Network/virtualNetworks/engineering-vnet-name/subnets/engineering-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"14379f8b1b31a189250e481bc9add46fac24422e7df58458d3083fada337892b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"78641ea7605e7bdbb487cc976b5506f47036a902e88b74d9c93c1803ee5fb81f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-wwzmnt\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d4c620\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-wwzmnt: failed waiting for cluster=\"cp-ystream-upgrade-4-23-wwzmnt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-snbqwtgczjvt\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a895c60-e356-43c7-bca0-9c5a3b86cb5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a895c60-e356-43c7-bca0-9c5a3b86cb5e\",\n \"name\": \"3a895c60-e356-43c7-bca0-9c5a3b86cb5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:09.344447699Z\",\n \"endTime\": \"2026-09-29T10:20:52.421684346Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-snbqwtgczjvt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-wwzmnt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-snbqwtgczjvt/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-wwzmnt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-snbqwtgczjvt/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-wwzmnt/subnets/customer-vnet-subnet-cp-ystream-wwzmnt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-wwzmnt\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d4c620\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-wwzmnt: failed waiting for cluster=\"cp-ystream-upgrade-4-23-wwzmnt\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-snbqwtgczjvt\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a895c60-e356-43c7-bca0-9c5a3b86cb5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a895c60-e356-43c7-bca0-9c5a3b86cb5e\",\n \"name\": \"3a895c60-e356-43c7-bca0-9c5a3b86cb5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:09.344447699Z\",\n \"endTime\": \"2026-09-29T10:20:52.421684346Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-snbqwtgczjvt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-wwzmnt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-snbqwtgczjvt/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-wwzmnt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-23-snbqwtgczjvt/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-wwzmnt/subnets/customer-vnet-subnet-cp-ystream-wwzmnt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-43/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"28586581d67ba6f62209036a65e8fad094be3a52312dbb3424ce9b318936f76b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"alert","signature_id":"2cef1ba7969aeef3f5b953642871fdd7bd7c01e09dbb042396f762cad2dd2662","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors6h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T10:44:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors6h30m\", cluster=\"ci01-j1689472-svc\", component=\"slo\", long_window=\"6h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1689472/providers/microsoft.monitor/accounts/services-j1689472\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"cluster-provision-errors\"\n Description: More than 30% of completed cluster create (install) operations on ci01-j1689472-svc failed over the last 6 hours (at least 5 completions), a 6x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors6h30m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors6h30m fired"}},{"row_id":"3481cb63b45375d071eff722cf2da5e3a222bec09d315a86946a9753bc455466","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"44a817fdb95148f5a6bd73ed22358c11d3d05bcfc3882c091495a70949a898bb","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00143e720\u003e: \n failed to create HCP cluster np-downgrade-z9nk67: failed waiting for cluster=\"np-downgrade-z9nk67\" in resourcegroup=\"rg-np-downgrade-z9nk67-s4q84qfkhnb7\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d22ba176-cd75-463b-bd75-84338c9fec78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d22ba176-cd75-463b-bd75-84338c9fec78\",\n \"name\": \"d22ba176-cd75-463b-bd75-84338c9fec78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:27:01.868359666Z\",\n \"endTime\": \"2026-09-29T10:37:32.769283352Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z9nk67-s4q84qfkhnb7/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-dg-z9nk67\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z9nk67-s4q84qfkhnb7/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-dg-z9nk67\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z9nk67-s4q84qfkhnb7/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-dg-z9nk67/subnets/customer-vnet-subnet-np-dg-z9nk67\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00143e720\u003e: \n failed to create HCP cluster np-downgrade-z9nk67: failed waiting for cluster=\"np-downgrade-z9nk67\" in resourcegroup=\"rg-np-downgrade-z9nk67-s4q84qfkhnb7\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d22ba176-cd75-463b-bd75-84338c9fec78\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d22ba176-cd75-463b-bd75-84338c9fec78\",\n \"name\": \"d22ba176-cd75-463b-bd75-84338c9fec78\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:27:01.868359666Z\",\n \"endTime\": \"2026-09-29T10:37:32.769283352Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z9nk67-s4q84qfkhnb7/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-dg-z9nk67\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z9nk67-s4q84qfkhnb7/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-dg-z9nk67\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-z9nk67-s4q84qfkhnb7/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-dg-z9nk67/subnets/customer-vnet-subnet-np-dg-z9nk67\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-34/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"395bdbab52555811bb1f7b12c5e22b264ae923d4f532a61b084800da75849c69","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"9e9ef341335ee3145b5fde11d0ebfd20b1ca2957f017e00f0eabffffaa41affd","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ee880\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-9drctqkfll54\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/93849e9c-7835-44fb-bc7a-3e4b4f9f789c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/93849e9c-7835-44fb-bc7a-3e4b4f9f789c\",\n \"name\": \"93849e9c-7835-44fb-bc7a-3e4b4f9f789c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:27:33.688719071Z\",\n \"endTime\": \"2026-09-29T10:38:22.768258871Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9drctqkfll54/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9drctqkfll54/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9drctqkfll54/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:227]: failed to create HCP cluster oidc-wi-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005ee880\u003e: \n failed to create HCP cluster oidc-wi-cluster: failed waiting for cluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-9drctqkfll54\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/93849e9c-7835-44fb-bc7a-3e4b4f9f789c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/93849e9c-7835-44fb-bc7a-3e4b4f9f789c\",\n \"name\": \"93849e9c-7835-44fb-bc7a-3e4b4f9f789c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:27:33.688719071Z\",\n \"endTime\": \"2026-09-29T10:38:22.768258871Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9drctqkfll54/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9drctqkfll54/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/oidc-wi-9drctqkfll54/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"3be3b5727a689ce622b2315ede68b28aba623b51cece984bf62e54c555c08f9a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"67ba5a784246260a679aac9904ae68181ba5093069f252245cda70c8c187b398","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-6shptl\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102a120\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-6shptl: failed waiting for cluster=\"cp-ystream-upgrade-4-22-6shptl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-25smmspmtfcv\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4ba6999-b11e-4f3a-b9b6-f1680ef46015\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4ba6999-b11e-4f3a-b9b6-f1680ef46015\",\n \"name\": \"d4ba6999-b11e-4f3a-b9b6-f1680ef46015\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:09.919955155Z\",\n \"endTime\": \"2026-09-29T10:15:41.486324304Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-25smmspmtfcv/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-6shptl\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-25smmspmtfcv/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-6shptl\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-25smmspmtfcv/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-6shptl/subnets/customer-vnet-subnet-cp-ystream-6shptl\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-6shptl\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00102a120\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-6shptl: failed waiting for cluster=\"cp-ystream-upgrade-4-22-6shptl\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-25smmspmtfcv\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4ba6999-b11e-4f3a-b9b6-f1680ef46015\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d4ba6999-b11e-4f3a-b9b6-f1680ef46015\",\n \"name\": \"d4ba6999-b11e-4f3a-b9b6-f1680ef46015\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:09.919955155Z\",\n \"endTime\": \"2026-09-29T10:15:41.486324304Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-25smmspmtfcv/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-6shptl\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-25smmspmtfcv/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-6shptl\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-22-25smmspmtfcv/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-6shptl/subnets/customer-vnet-subnet-cp-ystream-6shptl\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-42/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"3dabd9e38b02ae6b73661ca2ef0342869d2331c095a67bf88062095d07bd4cb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"62ac233f1ab000c79c3cd5089218bf4f3df62d96357a3d2a1321c8c792cfa721","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00150ca80\u003e: \n GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f65e33d-454e-4081-a989-7fbb802538e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f65e33d-454e-4081-a989-7fbb802538e0\",\n \"name\": \"8f65e33d-454e-4081-a989-7fbb802538e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:58.419342653Z\",\n \"endTime\": \"2026-09-29T10:16:52.273775387Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-52vvxmr4czww/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-52vvxmr4czww/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-52vvxmr4czww/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:131]: failed to wait for first cluster \"basic-hcp-cluster\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00150ca80\u003e: \n GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f65e33d-454e-4081-a989-7fbb802538e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8f65e33d-454e-4081-a989-7fbb802538e0\",\n \"name\": \"8f65e33d-454e-4081-a989-7fbb802538e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:58.419342653Z\",\n \"endTime\": \"2026-09-29T10:16:52.273775387Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-52vvxmr4czww/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-52vvxmr4czww/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-52vvxmr4czww/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-05/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"41763675430eb142563c0055c9f16cdb092d97875997fa097004727bcaa902ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"24f77691baac1004e264aeccc101fa944467fdb5bed9419e002588d64f24ab77","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-kvmknc at version 4.22.16\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce4920\u003e: \n failed to create HCP cluster np-skip-minor-kvmknc: failed waiting for cluster=\"np-skip-minor-kvmknc\" in resourcegroup=\"rg-np-skip-minor-kvmknc-dz8lc686d7rt\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81d6be9-f4fc-458e-8d78-4bd0c07395d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81d6be9-f4fc-458e-8d78-4bd0c07395d2\",\n \"name\": \"e81d6be9-f4fc-458e-8d78-4bd0c07395d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:26:32.346637677Z\",\n \"endTime\": \"2026-09-29T10:37:22.795520828Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kvmknc-dz8lc686d7rt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-skip-kvmknc\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kvmknc-dz8lc686d7rt/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-skip-kvmknc\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kvmknc-dz8lc686d7rt/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-skip-kvmknc/subnets/customer-vnet-subnet-np-skip-kvmknc\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-kvmknc at version 4.22.16\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce4920\u003e: \n failed to create HCP cluster np-skip-minor-kvmknc: failed waiting for cluster=\"np-skip-minor-kvmknc\" in resourcegroup=\"rg-np-skip-minor-kvmknc-dz8lc686d7rt\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81d6be9-f4fc-458e-8d78-4bd0c07395d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e81d6be9-f4fc-458e-8d78-4bd0c07395d2\",\n \"name\": \"e81d6be9-f4fc-458e-8d78-4bd0c07395d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:26:32.346637677Z\",\n \"endTime\": \"2026-09-29T10:37:22.795520828Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kvmknc-dz8lc686d7rt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-skip-kvmknc\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kvmknc-dz8lc686d7rt/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-skip-kvmknc\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-kvmknc-dz8lc686d7rt/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-skip-kvmknc/subnets/customer-vnet-subnet-np-skip-kvmknc\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-49/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"451dfd38b844b1049e3b403dbe8b020b5116e6810dd4789146d850bccd2aac98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"35a80301a0a589c8b7bc46f385eb33a3d2f9812dbf0f6c26f41f5b785f953d33","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8ace0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-nz7q47dlpkrd\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc94227d-1d5d-4a1a-9f66-728b95167ff1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc94227d-1d5d-4a1a-9f66-728b95167ff1\",\n \"name\": \"dc94227d-1d5d-4a1a-9f66-728b95167ff1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:31.176203601Z\",\n \"endTime\": \"2026-09-29T10:16:11.361244299Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nz7q47dlpkrd/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nz7q47dlpkrd/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nz7q47dlpkrd/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8ace0\u003e: \n failed to create HCP cluster ephemeral-disk: failed waiting for cluster=\"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-nz7q47dlpkrd\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc94227d-1d5d-4a1a-9f66-728b95167ff1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/dc94227d-1d5d-4a1a-9f66-728b95167ff1\",\n \"name\": \"dc94227d-1d5d-4a1a-9f66-728b95167ff1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:31.176203601Z\",\n \"endTime\": \"2026-09-29T10:16:11.361244299Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nz7q47dlpkrd/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nz7q47dlpkrd/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-nz7q47dlpkrd/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-57/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"45767f44577f0f7dfb0b8f28bc294dbcbe39db921a7220cec65b0f26a77ac3d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"5116d981025537b215477a824dd9f4d76d6de217536d1229d083b750a0f6ed8a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-vghcgzqmc9xx/cluster-candidate-5-0-sjlgzm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001126240\u003e: \n failed to create HCP cluster cluster-candidate-5-0-sjlgzm: failed waiting for cluster=\"cluster-candidate-5-0-sjlgzm\" in resourcegroup=\"rg-candidate-5-0-vghcgzqmc9xx\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc8532e2-9ff4-49cb-a696-faab85e49401\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc8532e2-9ff4-49cb-a696-faab85e49401\",\n \"name\": \"bc8532e2-9ff4-49cb-a696-faab85e49401\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.140197729Z\",\n \"endTime\": \"2026-09-29T10:15:41.030016505Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vghcgzqmc9xx/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-sjlgzm\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vghcgzqmc9xx/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-sjlgzm\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vghcgzqmc9xx/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-sjlgzm/subnets/customer-vnet-subnet-candidate-sjlgzm\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-vghcgzqmc9xx/cluster-candidate-5-0-sjlgzm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001126240\u003e: \n failed to create HCP cluster cluster-candidate-5-0-sjlgzm: failed waiting for cluster=\"cluster-candidate-5-0-sjlgzm\" in resourcegroup=\"rg-candidate-5-0-vghcgzqmc9xx\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc8532e2-9ff4-49cb-a696-faab85e49401\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bc8532e2-9ff4-49cb-a696-faab85e49401\",\n \"name\": \"bc8532e2-9ff4-49cb-a696-faab85e49401\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.140197729Z\",\n \"endTime\": \"2026-09-29T10:15:41.030016505Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vghcgzqmc9xx/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-sjlgzm\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vghcgzqmc9xx/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-sjlgzm\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vghcgzqmc9xx/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-sjlgzm/subnets/customer-vnet-subnet-candidate-sjlgzm\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-39/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"47e83c7b5caae4941749dacabf7af52d2b03ab1d0051a3c87a3e366f263cac00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"376a2e45cb584aa6985017149c8d6f358c46a7668f9a96b9ed423a3e3b495b95","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017d2340\u003e: \n failed to create HCP cluster np-dg-minor-drfvp6: failed waiting for cluster=\"np-dg-minor-drfvp6\" in resourcegroup=\"rg-np-dg-minor-drfvp6-4m9m4fntwzpw\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ab32cd8-1e0e-4843-b645-c6a3cc0e688d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ab32cd8-1e0e-4843-b645-c6a3cc0e688d\",\n \"name\": \"7ab32cd8-1e0e-4843-b645-c6a3cc0e688d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:27:35.626049713Z\",\n \"endTime\": \"2026-09-29T10:38:12.813999204Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-drfvp6-4m9m4fntwzpw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-dgm-drfvp6\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-drfvp6-4m9m4fntwzpw/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-dgm-drfvp6\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-drfvp6-4m9m4fntwzpw/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-dgm-drfvp6/subnets/customer-vnet-subnet-np-dgm-drfvp6\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017d2340\u003e: \n failed to create HCP cluster np-dg-minor-drfvp6: failed waiting for cluster=\"np-dg-minor-drfvp6\" in resourcegroup=\"rg-np-dg-minor-drfvp6-4m9m4fntwzpw\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ab32cd8-1e0e-4843-b645-c6a3cc0e688d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ab32cd8-1e0e-4843-b645-c6a3cc0e688d\",\n \"name\": \"7ab32cd8-1e0e-4843-b645-c6a3cc0e688d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:27:35.626049713Z\",\n \"endTime\": \"2026-09-29T10:38:12.813999204Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-drfvp6-4m9m4fntwzpw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-dgm-drfvp6\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-drfvp6-4m9m4fntwzpw/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-dgm-drfvp6\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-drfvp6-4m9m4fntwzpw/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-dgm-drfvp6/subnets/customer-vnet-subnet-np-dgm-drfvp6\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-04/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"49adc0ff2bdb9dd1a18871701d6f9ccdcfd0acfd2085480a1265fa69cbfdd4f7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"80425453a66191ce41935f70ee73d5471ecdd610b77707877961ba3af1a621a8","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad8ae0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-hd6dp5sbgq49\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/032bd3c1-7fb3-44b0-9748-4a6084a6ce28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/032bd3c1-7fb3-44b0-9748-4a6084a6ce28\",\n \"name\": \"032bd3c1-7fb3-44b0-9748-4a6084a6ce28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:21.550236296Z\",\n \"endTime\": \"2026-09-29T10:16:11.183490556Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-hd6dp5sbgq49/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-hd6dp5sbgq49/providers/Microsoft.Network/virtualNetworks/sre-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-hd6dp5sbgq49/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad8ae0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-hd6dp5sbgq49\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/032bd3c1-7fb3-44b0-9748-4a6084a6ce28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/032bd3c1-7fb3-44b0-9748-4a6084a6ce28\",\n \"name\": \"032bd3c1-7fb3-44b0-9748-4a6084a6ce28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:21.550236296Z\",\n \"endTime\": \"2026-09-29T10:16:11.183490556Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-hd6dp5sbgq49/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-hd6dp5sbgq49/providers/Microsoft.Network/virtualNetworks/sre-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-bootdiag-hd6dp5sbgq49/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-00/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"513d78cc37ebd87d9daef216b137a5a7f4c34e10c0ac0be066c0251ea4d23ec7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"d4a7f0ff9250f836801c608c34d4f26a01655c51d27f8bd715581a70c1e7643a","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:96]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000948720\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-x5frt4qn4msb\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46d56f94-f6d3-43fb-94a1-7c92e4dd83a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46d56f94-f6d3-43fb-94a1-7c92e4dd83a6\",\n \"name\": \"46d56f94-f6d3-43fb-94a1-7c92e4dd83a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:19.978633527Z\",\n \"endTime\": \"2026-09-29T10:15:52.776496615Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-x5frt4qn4msb/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-x5frt4qn4msb/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-x5frt4qn4msb/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:96]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000948720\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-x5frt4qn4msb\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46d56f94-f6d3-43fb-94a1-7c92e4dd83a6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46d56f94-f6d3-43fb-94a1-7c92e4dd83a6\",\n \"name\": \"46d56f94-f6d3-43fb-94a1-7c92e4dd83a6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:19.978633527Z\",\n \"endTime\": \"2026-09-29T10:15:52.776496615Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-x5frt4qn4msb/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-x5frt4qn4msb/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kms-key-rotate-x5frt4qn4msb/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-51/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"528d96227b66273d155973bc245dea940a92c557f8a4c3c6659cc2177677209e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"4ea5b3ae5dd9f669a29f18399124f77398af584a5fef16ed64367e250694b982","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--dlc69h84j7d7/cluster-hs-pre-dd87tz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f78460\u003e: \n failed to create HCP cluster cluster-hs-pre-dd87tz: failed waiting for cluster=\"cluster-hs-pre-dd87tz\" in resourcegroup=\"rg-hs-pre--dlc69h84j7d7\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a808325-ee37-4a1f-9122-d2089b25291e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a808325-ee37-4a1f-9122-d2089b25291e\",\n \"name\": \"7a808325-ee37-4a1f-9122-d2089b25291e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:51.001765736Z\",\n \"endTime\": \"2026-09-29T10:14:41.375660855Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--dlc69h84j7d7/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-hs-pre-dd87tz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--dlc69h84j7d7/providers/Microsoft.Network/virtualNetworks/customer-vnet-hs-pre-dd87tz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--dlc69h84j7d7/providers/Microsoft.Network/virtualNetworks/customer-vnet-hs-pre-dd87tz/subnets/customer-vnet-subnet-hs-pre-dd87tz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--dlc69h84j7d7/cluster-hs-pre-dd87tz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f78460\u003e: \n failed to create HCP cluster cluster-hs-pre-dd87tz: failed waiting for cluster=\"cluster-hs-pre-dd87tz\" in resourcegroup=\"rg-hs-pre--dlc69h84j7d7\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a808325-ee37-4a1f-9122-d2089b25291e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a808325-ee37-4a1f-9122-d2089b25291e\",\n \"name\": \"7a808325-ee37-4a1f-9122-d2089b25291e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:51.001765736Z\",\n \"endTime\": \"2026-09-29T10:14:41.375660855Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--dlc69h84j7d7/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-hs-pre-dd87tz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--dlc69h84j7d7/providers/Microsoft.Network/virtualNetworks/customer-vnet-hs-pre-dd87tz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--dlc69h84j7d7/providers/Microsoft.Network/virtualNetworks/customer-vnet-hs-pre-dd87tz/subnets/customer-vnet-subnet-hs-pre-dd87tz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-29/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"53ab7c2a39c7ae78f52d46cf0430a434b6c7317543fa5a7865ec00f986376ff9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"ef502afdccec5e3e6db5848ac2ba5cae769b27d1c94caff1ef2577ee286c98f6","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-xz9g6k\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00077c260\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-xz9g6k: failed waiting for cluster=\"cp-ystream-upgrade-5-1-xz9g6k\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-c8x5mjs5cfft\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7716933-f6cf-40c6-a133-f74cd998b029\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7716933-f6cf-40c6-a133-f74cd998b029\",\n \"name\": \"f7716933-f6cf-40c6-a133-f74cd998b029\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:13.237091558Z\",\n \"endTime\": \"2026-09-29T10:15:52.709600734Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-c8x5mjs5cfft/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-xz9g6k\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-c8x5mjs5cfft/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-xz9g6k\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-c8x5mjs5cfft/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-xz9g6k/subnets/customer-vnet-subnet-cp-ystream-xz9g6k\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-1-xz9g6k\" at install version 5.0\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00077c260\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-1-xz9g6k: failed waiting for cluster=\"cp-ystream-upgrade-5-1-xz9g6k\" in resourcegroup=\"rg-cp-ystream-upgrade-5-1-c8x5mjs5cfft\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7716933-f6cf-40c6-a133-f74cd998b029\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f7716933-f6cf-40c6-a133-f74cd998b029\",\n \"name\": \"f7716933-f6cf-40c6-a133-f74cd998b029\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:13.237091558Z\",\n \"endTime\": \"2026-09-29T10:15:52.709600734Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-c8x5mjs5cfft/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-xz9g6k\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-c8x5mjs5cfft/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-xz9g6k\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-1-c8x5mjs5cfft/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-xz9g6k/subnets/customer-vnet-subnet-cp-ystream-xz9g6k\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-45/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"57ed571bd03771cbc3ac1545e8effac11d3b749296ac4f7ad3f28bf6ff6f433f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"1702836eacbfab064686a4c82a79c2e6d3bf9ed9a090c4f5677545140fe2f29f","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011a5360\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-pp69lm2b6vl6\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/34e38587-d21d-4c25-a43c-426c54978cab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/34e38587-d21d-4c25-a43c-426c54978cab\",\n \"name\": \"34e38587-d21d-4c25-a43c-426c54978cab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:37.611341077Z\",\n \"endTime\": \"2026-09-29T10:15:01.194626974Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-metrics-pp69lm2b6vl6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-metrics-pp69lm2b6vl6/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-metrics-pp69lm2b6vl6/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011a5360\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-pp69lm2b6vl6\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/34e38587-d21d-4c25-a43c-426c54978cab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/34e38587-d21d-4c25-a43c-426c54978cab\",\n \"name\": \"34e38587-d21d-4c25-a43c-426c54978cab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:37.611341077Z\",\n \"endTime\": \"2026-09-29T10:15:01.194626974Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-metrics-pp69lm2b6vl6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-metrics-pp69lm2b6vl6/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-metrics-pp69lm2b6vl6/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-52/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"594a7adeaf612459bedb8c5a84e1631a695e93d09a58bb87abe93f2d3462f3a5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"c42cf87d27ef5f79c078adcc1f0fbdd1dcd1de3a610cbb96ec0abcfb54a819aa","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffe9c0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5f4jjlcqh7k2\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/78f11d63-e78a-4d03-ac4f-de0804b9c410\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/78f11d63-e78a-4d03-ac4f-de0804b9c410\",\n \"name\": \"78f11d63-e78a-4d03-ac4f-de0804b9c410\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.166862037Z\",\n \"endTime\": \"2026-09-29T10:13:53.607966146Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-5f4jjlcqh7k2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-5f4jjlcqh7k2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-5f4jjlcqh7k2/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffe9c0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-5f4jjlcqh7k2\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/78f11d63-e78a-4d03-ac4f-de0804b9c410\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/78f11d63-e78a-4d03-ac4f-de0804b9c410\",\n \"name\": \"78f11d63-e78a-4d03-ac4f-de0804b9c410\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.166862037Z\",\n \"endTime\": \"2026-09-29T10:13:53.607966146Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-5f4jjlcqh7k2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-5f4jjlcqh7k2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-5f4jjlcqh7k2/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-19/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"5a74521e14d24361984c7f52f3dbfd19ebbfd61c112e2d4da05150faa4b9f332","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"922482905d9f753fc55638928625f28a60ac1d4c9bc3953857c4c7c9916d9a3c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-5zbw47rdl4mr/cluster-candidate-4-20-qjpkqz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001328720\u003e: \n failed to create HCP cluster cluster-candidate-4-20-qjpkqz: failed waiting for cluster=\"cluster-candidate-4-20-qjpkqz\" in resourcegroup=\"rg-candidate-4-20-5zbw47rdl4mr\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e44b249-5b1e-423c-aba0-9e48b1f8f1c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e44b249-5b1e-423c-aba0-9e48b1f8f1c1\",\n \"name\": \"5e44b249-5b1e-423c-aba0-9e48b1f8f1c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:24.083834849Z\",\n \"endTime\": \"2026-09-29T10:14:41.419950172Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-5zbw47rdl4mr/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-qjpkqz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-5zbw47rdl4mr/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-qjpkqz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-5zbw47rdl4mr/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-qjpkqz/subnets/customer-vnet-subnet-candidate-qjpkqz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-5zbw47rdl4mr/cluster-candidate-4-20-qjpkqz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001328720\u003e: \n failed to create HCP cluster cluster-candidate-4-20-qjpkqz: failed waiting for cluster=\"cluster-candidate-4-20-qjpkqz\" in resourcegroup=\"rg-candidate-4-20-5zbw47rdl4mr\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e44b249-5b1e-423c-aba0-9e48b1f8f1c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5e44b249-5b1e-423c-aba0-9e48b1f8f1c1\",\n \"name\": \"5e44b249-5b1e-423c-aba0-9e48b1f8f1c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:24.083834849Z\",\n \"endTime\": \"2026-09-29T10:14:41.419950172Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-5zbw47rdl4mr/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-qjpkqz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-5zbw47rdl4mr/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-qjpkqz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-5zbw47rdl4mr/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-qjpkqz/subnets/customer-vnet-subnet-candidate-qjpkqz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-32/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"5acb14e8ed5c0a7f20946dd90e7ebab010f17488c3492bb8bff265e23d7dcf04","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"c6011225e00fbeca8f5dac2990f50ceba63410ea92b30545a88ad7c9e0584b16","test_name":"Customer should provision Azure Disk and Azure File CSI volumes and run a pod against each","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_csi_persistent_volume.go:82]: HCP cluster rg-csi--dg2qhg6bg8n9/cluster-csi-4ktxkk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0c3c0\u003e: \n failed to create HCP cluster cluster-csi-4ktxkk: failed waiting for cluster=\"cluster-csi-4ktxkk\" in resourcegroup=\"rg-csi--dg2qhg6bg8n9\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2956d08-6fdb-4fc4-8d83-c9e6d66369de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2956d08-6fdb-4fc4-8d83-c9e6d66369de\",\n \"name\": \"c2956d08-6fdb-4fc4-8d83-c9e6d66369de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:00.530843739Z\",\n \"endTime\": \"2026-09-29T10:15:31.876444705Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-csi--dg2qhg6bg8n9/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-csi-4ktxkk\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-csi--dg2qhg6bg8n9/providers/Microsoft.Network/virtualNetworks/customer-vnet-csi-4ktxkk\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-csi--dg2qhg6bg8n9/providers/Microsoft.Network/virtualNetworks/customer-vnet-csi-4ktxkk/subnets/customer-vnet-subnet-csi-4ktxkk\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_csi_persistent_volume.go:82]: HCP cluster rg-csi--dg2qhg6bg8n9/cluster-csi-4ktxkk should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0c3c0\u003e: \n failed to create HCP cluster cluster-csi-4ktxkk: failed waiting for cluster=\"cluster-csi-4ktxkk\" in resourcegroup=\"rg-csi--dg2qhg6bg8n9\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2956d08-6fdb-4fc4-8d83-c9e6d66369de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c2956d08-6fdb-4fc4-8d83-c9e6d66369de\",\n \"name\": \"c2956d08-6fdb-4fc4-8d83-c9e6d66369de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:00.530843739Z\",\n \"endTime\": \"2026-09-29T10:15:31.876444705Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-csi--dg2qhg6bg8n9/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-csi-4ktxkk\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-csi--dg2qhg6bg8n9/providers/Microsoft.Network/virtualNetworks/customer-vnet-csi-4ktxkk\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-csi--dg2qhg6bg8n9/providers/Microsoft.Network/virtualNetworks/customer-vnet-csi-4ktxkk/subnets/customer-vnet-subnet-csi-4ktxkk\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-28/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"5ef66c3e07bdc0cf8317704422c9f7558a21f3026692744871be22a97fec4e6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"5ad678d745e22693545052dd62977a43a3b07fa8ca809dfd5000dd0215ec03f4","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f2a680\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-65hr2kns66tm\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5233027-64b1-43d7-a1cd-05dbfcbd0d75\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5233027-64b1-43d7-a1cd-05dbfcbd0d75\",\n \"name\": \"b5233027-64b1-43d7-a1cd-05dbfcbd0d75\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:09.797407753Z\",\n \"endTime\": \"2026-09-29T10:15:32.034214269Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-65hr2kns66tm/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-65hr2kns66tm/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-65hr2kns66tm/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f2a680\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-65hr2kns66tm\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5233027-64b1-43d7-a1cd-05dbfcbd0d75\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b5233027-64b1-43d7-a1cd-05dbfcbd0d75\",\n \"name\": \"b5233027-64b1-43d7-a1cd-05dbfcbd0d75\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:09.797407753Z\",\n \"endTime\": \"2026-09-29T10:15:32.034214269Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-65hr2kns66tm/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-65hr2kns66tm/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-65hr2kns66tm/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-11/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"62b9d233dbc43736b3ca459e2792109fa54fd2a923185c4af7b2e772318d39c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"97240d8885a0bdfc86afa503aac43fee59507e22b5387aeb628e8d3e06faee04","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.004s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"678c84558c223037736893529792ac73bcebe5089ac400b28e46784edceac9b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"27bf4f211d95c9d95303c9cba12067463dc919319f47d2e622951d441f24925b","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010481e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xq7qjk96g7jx\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/693c0a68-3803-4693-80d3-752a0f91b68e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/693c0a68-3803-4693-80d3-752a0f91b68e\",\n \"name\": \"693c0a68-3803-4693-80d3-752a0f91b68e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:52.514376366Z\",\n \"endTime\": \"2026-09-29T10:14:01.36319585Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-xq7qjk96g7jx/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-xq7qjk96g7jx/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-xq7qjk96g7jx/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010481e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xq7qjk96g7jx\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/693c0a68-3803-4693-80d3-752a0f91b68e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/693c0a68-3803-4693-80d3-752a0f91b68e\",\n \"name\": \"693c0a68-3803-4693-80d3-752a0f91b68e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:52.514376366Z\",\n \"endTime\": \"2026-09-29T10:14:01.36319585Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-xq7qjk96g7jx/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-xq7qjk96g7jx/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-xq7qjk96g7jx/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-26/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"6a56ee0bf31a9ce7f049af96e61cc3c36edb969c1fc38ce617b533896ae4c8d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"78b3ae02fbf4c379addd1fba28ebc307a7aedf4b90944426c6d3800c6b40d199","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc660\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-c7dgbn6wxjd4\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fe36d445-12f5-42cf-87f6-9addf47f57a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fe36d445-12f5-42cf-87f6-9addf47f57a2\",\n \"name\": \"fe36d445-12f5-42cf-87f6-9addf47f57a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:10.892880097Z\",\n \"endTime\": \"2026-09-29T10:16:31.221473624Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-c7dgbn6wxjd4/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-c7dgbn6wxjd4/providers/Microsoft.Network/virtualNetworks/sre-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-c7dgbn6wxjd4/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc660\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-c7dgbn6wxjd4\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fe36d445-12f5-42cf-87f6-9addf47f57a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/fe36d445-12f5-42cf-87f6-9addf47f57a2\",\n \"name\": \"fe36d445-12f5-42cf-87f6-9addf47f57a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:10.892880097Z\",\n \"endTime\": \"2026-09-29T10:16:31.221473624Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-c7dgbn6wxjd4/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-c7dgbn6wxjd4/providers/Microsoft.Network/virtualNetworks/sre-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-serialconsole-c7dgbn6wxjd4/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-01/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"6b86ff161c5f8e7611e37b2fbeccf75edcd17044e88328bc06118bbab9aada0c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"4c579176945b4d04dca6c79000c3ae6259786d4e4db01668f42f9421fa4fdfbb","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-4v6kfg with version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001600780\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-4v6kfg: failed waiting for cluster=\"np-version-upgrade-cluster-4v6kfg\" in resourcegroup=\"rg-np-version-upgrade-4v6kfg-dhkclrrktxtw\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2e1b61f6-3310-4138-b334-decb8675592f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2e1b61f6-3310-4138-b334-decb8675592f\",\n \"name\": \"2e1b61f6-3310-4138-b334-decb8675592f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:26:31.535924331Z\",\n \"endTime\": \"2026-09-29T10:37:12.843846164Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4v6kfg-dhkclrrktxtw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-4v6kfg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4v6kfg-dhkclrrktxtw/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-4v6kfg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4v6kfg-dhkclrrktxtw/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-4v6kfg/subnets/customer-vnet-subnet-np-upgrade-4v6kfg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-4v6kfg with version 4.20.40\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001600780\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-4v6kfg: failed waiting for cluster=\"np-version-upgrade-cluster-4v6kfg\" in resourcegroup=\"rg-np-version-upgrade-4v6kfg-dhkclrrktxtw\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2e1b61f6-3310-4138-b334-decb8675592f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2e1b61f6-3310-4138-b334-decb8675592f\",\n \"name\": \"2e1b61f6-3310-4138-b334-decb8675592f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:26:31.535924331Z\",\n \"endTime\": \"2026-09-29T10:37:12.843846164Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4v6kfg-dhkclrrktxtw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-4v6kfg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4v6kfg-dhkclrrktxtw/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-4v6kfg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-4v6kfg-dhkclrrktxtw/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-4v6kfg/subnets/customer-vnet-subnet-np-upgrade-4v6kfg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"763e2a123775f8441dd8b4c82a27c8a8ecdf6c8bdcf85f6382bfa2f571daef2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"5a31de2cad2d4f3bde532db578eed488cfc2363bde524afbac38db13cc2fd46c","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100c880\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5s4r4xbq2lds\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/00b0e524-55c1-4e8d-ad3f-dec51f7a8659\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/00b0e524-55c1-4e8d-ad3f-dec51f7a8659\",\n \"name\": \"00b0e524-55c1-4e8d-ad3f-dec51f7a8659\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:27.721396126Z\",\n \"endTime\": \"2026-09-29T10:15:52.414444727Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-5s4r4xbq2lds/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-5s4r4xbq2lds/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-5s4r4xbq2lds/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100c880\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-5s4r4xbq2lds\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/00b0e524-55c1-4e8d-ad3f-dec51f7a8659\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/00b0e524-55c1-4e8d-ad3f-dec51f7a8659\",\n \"name\": \"00b0e524-55c1-4e8d-ad3f-dec51f7a8659\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:27.721396126Z\",\n \"endTime\": \"2026-09-29T10:15:52.414444727Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-5s4r4xbq2lds/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-5s4r4xbq2lds/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/disabled-image-registry-5s4r4xbq2lds/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-50/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"7771a8c15bcc6ec51bcb2a711847d4649ab384f70e88f20a16d901b5ce494121","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"372c7cbc57e431768b23d6f16f5f4115d379685d8bab58afdd9e0026e37245fd","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffa860\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-zqslbgmb54td\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/12b2d350-c51d-443c-88a6-513008ab851e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/12b2d350-c51d-443c-88a6-513008ab851e\",\n \"name\": \"12b2d350-c51d-443c-88a6-513008ab851e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.712139084Z\",\n \"endTime\": \"2026-09-29T10:14:11.221242437Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-zqslbgmb54td/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-zqslbgmb54td/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-zqslbgmb54td/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ffa860\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-zqslbgmb54td\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/12b2d350-c51d-443c-88a6-513008ab851e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/12b2d350-c51d-443c-88a6-513008ab851e\",\n \"name\": \"12b2d350-c51d-443c-88a6-513008ab851e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.712139084Z\",\n \"endTime\": \"2026-09-29T10:14:11.221242437Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-zqslbgmb54td/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-zqslbgmb54td/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-zqslbgmb54td/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-24/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"791dcf9d5b1a7cef8cc5f787ed782b46ecc3ec16c77f758ad4b691d4e8d54dd5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"bd743ff2c0b4d9b09efa2db55d9262b750fbcd4443c46cf3423b91c864f9544c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-xpxzhs\" with version 4.21.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b54560\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xpxzhs: failed waiting for cluster=\"cluster-zstream-4-21-xpxzhs\" in resourcegroup=\"rg-zstream-upgrade-4-21-pvs6q95vgdld\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d73db90f-f0ed-4755-be1b-831256f47752\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d73db90f-f0ed-4755-be1b-831256f47752\",\n \"name\": \"d73db90f-f0ed-4755-be1b-831256f47752\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:00.654260263Z\",\n \"endTime\": \"2026-09-29T10:13:21.891407638Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-pvs6q95vgdld/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-xpxzhs\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-pvs6q95vgdld/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xpxzhs\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-pvs6q95vgdld/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xpxzhs/subnets/customer-vnet-subnet-zstream-xpxzhs\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-xpxzhs\" with version 4.21.34 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b54560\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xpxzhs: failed waiting for cluster=\"cluster-zstream-4-21-xpxzhs\" in resourcegroup=\"rg-zstream-upgrade-4-21-pvs6q95vgdld\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d73db90f-f0ed-4755-be1b-831256f47752\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d73db90f-f0ed-4755-be1b-831256f47752\",\n \"name\": \"d73db90f-f0ed-4755-be1b-831256f47752\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:00.654260263Z\",\n \"endTime\": \"2026-09-29T10:13:21.891407638Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-pvs6q95vgdld/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-xpxzhs\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-pvs6q95vgdld/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xpxzhs\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-pvs6q95vgdld/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xpxzhs/subnets/customer-vnet-subnet-zstream-xpxzhs\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-38/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"7c19754945ca524a9ff7fed985218597311030ee693133451d1327b638ac6575","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"7d2b869c5b5311dedf526d8a5202bd455af12b375a27fa2b53891bdf744d2300","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"759d392837953820de5a9533a6cd31588c8e0f68881cc7b577f44196cdaa4197","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d02160\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-q6c67smbbmls\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef1fea5f-3ed3-4a38-b562-df50a232f5bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef1fea5f-3ed3-4a38-b562-df50a232f5bd\",\n \"name\": \"ef1fea5f-3ed3-4a38-b562-df50a232f5bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:36.62441428Z\",\n \"endTime\": \"2026-09-29T10:12:41.175278829Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-q6c67smbbmls/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-q6c67smbbmls/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-q6c67smbbmls/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:291]: failed to create HCP cluster node-limit-cluster with MaxNodesTotal limit\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d02160\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-q6c67smbbmls\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef1fea5f-3ed3-4a38-b562-df50a232f5bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ef1fea5f-3ed3-4a38-b562-df50a232f5bd\",\n \"name\": \"ef1fea5f-3ed3-4a38-b562-df50a232f5bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:36.62441428Z\",\n \"endTime\": \"2026-09-29T10:12:41.175278829Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-q6c67smbbmls/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-q6c67smbbmls/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-q6c67smbbmls/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-54/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"7fddb425eb8585c84ad71632c9a28693f2d9b2f8a7bfe0064d003f16839c5d64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"7e1072ad23a75e87b713b81eb9520a7e8ede407604f2d94f34708dbd39dd27e2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd0480\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-pv4qfkswpfmf\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70ce7ba8-c4a1-40a3-80dd-cc0fc2fb35ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70ce7ba8-c4a1-40a3-80dd-cc0fc2fb35ee\",\n \"name\": \"70ce7ba8-c4a1-40a3-80dd-cc0fc2fb35ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.348120711Z\",\n \"endTime\": \"2026-09-29T10:14:01.441483804Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-pv4qfkswpfmf/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-pv4qfkswpfmf/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-pv4qfkswpfmf/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd0480\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-pv4qfkswpfmf\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70ce7ba8-c4a1-40a3-80dd-cc0fc2fb35ee\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70ce7ba8-c4a1-40a3-80dd-cc0fc2fb35ee\",\n \"name\": \"70ce7ba8-c4a1-40a3-80dd-cc0fc2fb35ee\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.348120711Z\",\n \"endTime\": \"2026-09-29T10:14:01.441483804Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-pv4qfkswpfmf/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-pv4qfkswpfmf/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-pv4qfkswpfmf/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-15/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"85493ef784cb9edeba7fd079e80dd961e51a2b6fcc66d3205978c669cd45b9c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"2a0922766a2d45c3a7caa61f9f3addbbf7a0931628bdafa9418797df42286c15","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-f46qrt\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016a8a20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-f46qrt: failed waiting for cluster=\"cp-ystream-upgrade-5-0-f46qrt\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-wph94n87t72f\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96efa66f-2606-447f-bf6e-a235fb8f6225\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96efa66f-2606-447f-bf6e-a235fb8f6225\",\n \"name\": \"96efa66f-2606-447f-bf6e-a235fb8f6225\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:07.575904604Z\",\n \"endTime\": \"2026-09-29T10:13:11.218207283Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-wph94n87t72f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-f46qrt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-wph94n87t72f/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-f46qrt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-wph94n87t72f/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-f46qrt/subnets/customer-vnet-subnet-cp-ystream-f46qrt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-f46qrt\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016a8a20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-f46qrt: failed waiting for cluster=\"cp-ystream-upgrade-5-0-f46qrt\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-wph94n87t72f\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96efa66f-2606-447f-bf6e-a235fb8f6225\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/96efa66f-2606-447f-bf6e-a235fb8f6225\",\n \"name\": \"96efa66f-2606-447f-bf6e-a235fb8f6225\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:07.575904604Z\",\n \"endTime\": \"2026-09-29T10:13:11.218207283Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-wph94n87t72f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-f46qrt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-wph94n87t72f/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-f46qrt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-5-0-wph94n87t72f/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-f46qrt/subnets/customer-vnet-subnet-cp-ystream-f46qrt\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-44/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"8801ed545252e42b04269bae446aca4685e8490e93c73e5b64597d3379642775","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"12c2e68779c1f85d69990aab5c7f1d938154d259af0c5a42d1b11ac37bfdc4a3","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004bebe0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-97m7h85kwhxv\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6d4af28-6362-4b96-8688-b8bad6577688\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6d4af28-6362-4b96-8688-b8bad6577688\",\n \"name\": \"a6d4af28-6362-4b96-8688-b8bad6577688\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.272112062Z\",\n \"endTime\": \"2026-09-29T10:13:31.323265401Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-97m7h85kwhxv/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-97m7h85kwhxv/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-97m7h85kwhxv/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004bebe0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-97m7h85kwhxv\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6d4af28-6362-4b96-8688-b8bad6577688\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a6d4af28-6362-4b96-8688-b8bad6577688\",\n \"name\": \"a6d4af28-6362-4b96-8688-b8bad6577688\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.272112062Z\",\n \"endTime\": \"2026-09-29T10:13:31.323265401Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-97m7h85kwhxv/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-97m7h85kwhxv/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-97m7h85kwhxv/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-13/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"8c75007f90ff3bcff1cf1765cb42c363e737e6538f2a50d87244eee3fe7786bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"e6588aebfdbe2f5dabc9bcfcb3ee5e7220cce1bf635fbf86b834713e47465f82","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009fc9a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-psskzmmndkw7\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185a1bb8-7e6c-4031-b767-15134b65c631\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185a1bb8-7e6c-4031-b767-15134b65c631\",\n \"name\": \"185a1bb8-7e6c-4031-b767-15134b65c631\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:10.870653422Z\",\n \"endTime\": \"2026-09-29T10:13:52.94070397Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-psskzmmndkw7/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-psskzmmndkw7/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-psskzmmndkw7/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009fc9a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-psskzmmndkw7\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185a1bb8-7e6c-4031-b767-15134b65c631\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/185a1bb8-7e6c-4031-b767-15134b65c631\",\n \"name\": \"185a1bb8-7e6c-4031-b767-15134b65c631\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:10.870653422Z\",\n \"endTime\": \"2026-09-29T10:13:52.94070397Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-psskzmmndkw7/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-psskzmmndkw7/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-psskzmmndkw7/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-25/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"8c79efcaf8acde7f7037f84d1ad0e18dc6a270d832e339b9191f90816cbcf824","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"196dd7c370a3d588c6cba1bbc0b5abc38dc3b03aa3baea8d4cea27999cb931cd","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-sms2ms\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010eed20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-sms2ms: failed waiting for cluster=\"cp-ystream-upgrade-4-21-sms2ms\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-szxp6m7wgk7k\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c9ec5ba5-f479-41e3-9581-278d760ba0f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c9ec5ba5-f479-41e3-9581-278d760ba0f4\",\n \"name\": \"c9ec5ba5-f479-41e3-9581-278d760ba0f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:39.694340785Z\",\n \"endTime\": \"2026-09-29T10:16:11.406693738Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-szxp6m7wgk7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-sms2ms\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-szxp6m7wgk7k/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-sms2ms\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-szxp6m7wgk7k/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-sms2ms/subnets/customer-vnet-subnet-cp-ystream-sms2ms\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-sms2ms\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010eed20\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-sms2ms: failed waiting for cluster=\"cp-ystream-upgrade-4-21-sms2ms\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-szxp6m7wgk7k\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c9ec5ba5-f479-41e3-9581-278d760ba0f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c9ec5ba5-f479-41e3-9581-278d760ba0f4\",\n \"name\": \"c9ec5ba5-f479-41e3-9581-278d760ba0f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:39.694340785Z\",\n \"endTime\": \"2026-09-29T10:16:11.406693738Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-szxp6m7wgk7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-cp-ystream-sms2ms\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-szxp6m7wgk7k/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-sms2ms\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cp-ystream-upgrade-4-21-szxp6m7wgk7k/providers/Microsoft.Network/virtualNetworks/customer-vnet-cp-ystream-sms2ms/subnets/customer-vnet-subnet-cp-ystream-sms2ms\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-41/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"8cfbb31cbd25aa0451bc2196bf4347698e7de30aefc61063dd31f790da5afbc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"9396df2c96fda371380dfecff5fd399f5bba7f965d90e0e08fe6d561d4e4c8db","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c98320\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-99xw9v2t7hsq\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46c52242-3496-4af2-9b45-044be88e80a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46c52242-3496-4af2-9b45-044be88e80a3\",\n \"name\": \"46c52242-3496-4af2-9b45-044be88e80a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:49.086506775Z\",\n \"endTime\": \"2026-09-29T10:16:52.91664711Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-99xw9v2t7hsq/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-99xw9v2t7hsq/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-99xw9v2t7hsq/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:92]: failed to create HCP cluster cordon-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c98320\u003e: \n failed to create HCP cluster cordon-hcp-cluster: failed waiting for cluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-99xw9v2t7hsq\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46c52242-3496-4af2-9b45-044be88e80a3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/46c52242-3496-4af2-9b45-044be88e80a3\",\n \"name\": \"46c52242-3496-4af2-9b45-044be88e80a3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:49.086506775Z\",\n \"endTime\": \"2026-09-29T10:16:52.91664711Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-99xw9v2t7hsq/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-99xw9v2t7hsq/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cordon-test-99xw9v2t7hsq/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-55/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"91a78f077d5a4162795ee9142b04518665e1ff9eb266126e76e87d0e74adb42b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"b3e40c85e3aca11d514e42b735b50705e92401a2125742c63d38e34875817eb7","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-xkvvsv\" with version 4.20.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011c4120\u003e: \n failed to create HCP cluster cluster-zstream-4-20-xkvvsv: failed waiting for cluster=\"cluster-zstream-4-20-xkvvsv\" in resourcegroup=\"rg-zstream-upgrade-4-20-6kzlnt8l9b5s\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f394eec-afec-47fe-ad57-43abb153b790\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f394eec-afec-47fe-ad57-43abb153b790\",\n \"name\": \"2f394eec-afec-47fe-ad57-43abb153b790\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:51.060434394Z\",\n \"endTime\": \"2026-09-29T10:13:11.391765541Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6kzlnt8l9b5s/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-xkvvsv\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6kzlnt8l9b5s/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xkvvsv\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6kzlnt8l9b5s/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xkvvsv/subnets/customer-vnet-subnet-zstream-xkvvsv\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-xkvvsv\" with version 4.20.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011c4120\u003e: \n failed to create HCP cluster cluster-zstream-4-20-xkvvsv: failed waiting for cluster=\"cluster-zstream-4-20-xkvvsv\" in resourcegroup=\"rg-zstream-upgrade-4-20-6kzlnt8l9b5s\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f394eec-afec-47fe-ad57-43abb153b790\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2f394eec-afec-47fe-ad57-43abb153b790\",\n \"name\": \"2f394eec-afec-47fe-ad57-43abb153b790\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:51.060434394Z\",\n \"endTime\": \"2026-09-29T10:13:11.391765541Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6kzlnt8l9b5s/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-xkvvsv\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6kzlnt8l9b5s/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xkvvsv\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-6kzlnt8l9b5s/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xkvvsv/subnets/customer-vnet-subnet-zstream-xkvvsv\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-35/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"96234846703d9be9fedb932264d8534eed9a9c7b177d28b260a1cf16a342cd30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"95a8b9242a90178c5d989070600badc086d81e180e5b829f40c3b2df8a0da4ed","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060f7e0\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-qq7ft86884v2\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9843cb0c-2cde-47ba-9d76-843d09218866\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9843cb0c-2cde-47ba-9d76-843d09218866\",\n \"name\": \"9843cb0c-2cde-47ba-9d76-843d09218866\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:42.104418446Z\",\n \"endTime\": \"2026-09-29T10:15:01.52884519Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-qq7ft86884v2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-qq7ft86884v2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-qq7ft86884v2/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060f7e0\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-qq7ft86884v2\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9843cb0c-2cde-47ba-9d76-843d09218866\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9843cb0c-2cde-47ba-9d76-843d09218866\",\n \"name\": \"9843cb0c-2cde-47ba-9d76-843d09218866\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:42.104418446Z\",\n \"endTime\": \"2026-09-29T10:15:01.52884519Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-qq7ft86884v2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-qq7ft86884v2/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-qq7ft86884v2/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-27/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"9a90c457f2c604d2f2d868ec550636695059f128ac91223929933c013e9df28a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"e47b369318980e641f71ba8cc1940d7690c25a686c60de62c23adda0c610c822","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-23-dtz88l\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb6bc0\u003e: \n failed to create HCP cluster cluster-zstream-4-23-dtz88l: failed waiting for cluster=\"cluster-zstream-4-23-dtz88l\" in resourcegroup=\"rg-zstream-upgrade-4-23-g4nptkqn9gc2\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf222f6d-8eee-41f8-ad5b-7d40987582f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf222f6d-8eee-41f8-ad5b-7d40987582f4\",\n \"name\": \"cf222f6d-8eee-41f8-ad5b-7d40987582f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:51.192329089Z\",\n \"endTime\": \"2026-09-29T10:12:41.378177782Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-g4nptkqn9gc2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-dtz88l\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-g4nptkqn9gc2/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-dtz88l\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-g4nptkqn9gc2/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-dtz88l/subnets/customer-vnet-subnet-zstream-dtz88l\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-23-dtz88l\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb6bc0\u003e: \n failed to create HCP cluster cluster-zstream-4-23-dtz88l: failed waiting for cluster=\"cluster-zstream-4-23-dtz88l\" in resourcegroup=\"rg-zstream-upgrade-4-23-g4nptkqn9gc2\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf222f6d-8eee-41f8-ad5b-7d40987582f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf222f6d-8eee-41f8-ad5b-7d40987582f4\",\n \"name\": \"cf222f6d-8eee-41f8-ad5b-7d40987582f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:51.192329089Z\",\n \"endTime\": \"2026-09-29T10:12:41.378177782Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-g4nptkqn9gc2/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-dtz88l\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-g4nptkqn9gc2/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-dtz88l\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-23-g4nptkqn9gc2/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-dtz88l/subnets/customer-vnet-subnet-zstream-dtz88l\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-31/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"9aaca0b53d19b4f7604487ec92581352a86789df17a6d373b20e1f07d22e66bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"c5a904faa6ff8b90d92a60b4364847ea676dd584aaaa19bf6e9a9a8c4189aaed","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vmqvmb with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011a68c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-vmqvmb: failed waiting for cluster=\"np-version-upgrade-cluster-vmqvmb\" in resourcegroup=\"rg-np-version-upgrade-vmqvmb-4bhzlc9lbv9z\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a6112a7-64f2-45e0-be4c-2348ac9125d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a6112a7-64f2-45e0-be4c-2348ac9125d2\",\n \"name\": \"0a6112a7-64f2-45e0-be4c-2348ac9125d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:23:33.515294991Z\",\n \"endTime\": \"2026-09-29T10:33:52.76910443Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-vmqvmb-4bhzlc9lbv9z/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-vmqvmb\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-vmqvmb-4bhzlc9lbv9z/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-vmqvmb\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-vmqvmb-4bhzlc9lbv9z/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-vmqvmb/subnets/customer-vnet-subnet-np-upgrade-vmqvmb\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-vmqvmb with version 4.21.35\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011a68c0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-vmqvmb: failed waiting for cluster=\"np-version-upgrade-cluster-vmqvmb\" in resourcegroup=\"rg-np-version-upgrade-vmqvmb-4bhzlc9lbv9z\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a6112a7-64f2-45e0-be4c-2348ac9125d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0a6112a7-64f2-45e0-be4c-2348ac9125d2\",\n \"name\": \"0a6112a7-64f2-45e0-be4c-2348ac9125d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:23:33.515294991Z\",\n \"endTime\": \"2026-09-29T10:33:52.76910443Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-vmqvmb-4bhzlc9lbv9z/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-vmqvmb\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-vmqvmb-4bhzlc9lbv9z/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-vmqvmb\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-vmqvmb-4bhzlc9lbv9z/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-upgrade-vmqvmb/subnets/customer-vnet-subnet-np-upgrade-vmqvmb\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"9f53601fe2120c75baec9723291e7c83953cfc252c5b7041c45f2e2d1d80a0bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"1a2494caf88ca5012988ba3b15f28cd52a68aa903956e36f8a58f512bcfa1c47","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac4280\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mg85jzxfhrd8\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9fb4856f-ea5f-41d9-b980-08fd41f94243\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9fb4856f-ea5f-41d9-b980-08fd41f94243\",\n \"name\": \"9fb4856f-ea5f-41d9-b980-08fd41f94243\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:26.901175658Z\",\n \"endTime\": \"2026-09-29T10:16:11.183823958Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-mg85jzxfhrd8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-mg85jzxfhrd8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-mg85jzxfhrd8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac4280\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-mg85jzxfhrd8\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9fb4856f-ea5f-41d9-b980-08fd41f94243\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9fb4856f-ea5f-41d9-b980-08fd41f94243\",\n \"name\": \"9fb4856f-ea5f-41d9-b980-08fd41f94243\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:26.901175658Z\",\n \"endTime\": \"2026-09-29T10:16:11.183823958Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-mg85jzxfhrd8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-mg85jzxfhrd8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-mg85jzxfhrd8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-46/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"a5f70807ba90073ffd00b9e48db0d6606dc61b71da722f85e7b456ae999f5894","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"4519e5b058d02cb66d24e9aa9357b0980feaa1eef0ca9340ee8651e0a6630c9d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-crbpnkmg9p4b/cluster-candidate-4-23-tv4mch should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001292440\u003e: \n failed to create HCP cluster cluster-candidate-4-23-tv4mch: failed waiting for cluster=\"cluster-candidate-4-23-tv4mch\" in resourcegroup=\"rg-candidate-4-23-crbpnkmg9p4b\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c89416c-d90d-4898-abe4-765f7ae4ca65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c89416c-d90d-4898-abe4-765f7ae4ca65\",\n \"name\": \"3c89416c-d90d-4898-abe4-765f7ae4ca65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.940790184Z\",\n \"endTime\": \"2026-09-29T10:14:01.502862071Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-crbpnkmg9p4b/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-tv4mch\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-crbpnkmg9p4b/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-tv4mch\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-crbpnkmg9p4b/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-tv4mch/subnets/customer-vnet-subnet-candidate-tv4mch\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-crbpnkmg9p4b/cluster-candidate-4-23-tv4mch should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001292440\u003e: \n failed to create HCP cluster cluster-candidate-4-23-tv4mch: failed waiting for cluster=\"cluster-candidate-4-23-tv4mch\" in resourcegroup=\"rg-candidate-4-23-crbpnkmg9p4b\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c89416c-d90d-4898-abe4-765f7ae4ca65\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c89416c-d90d-4898-abe4-765f7ae4ca65\",\n \"name\": \"3c89416c-d90d-4898-abe4-765f7ae4ca65\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.940790184Z\",\n \"endTime\": \"2026-09-29T10:14:01.502862071Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-crbpnkmg9p4b/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-tv4mch\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-crbpnkmg9p4b/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-tv4mch\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-crbpnkmg9p4b/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-tv4mch/subnets/customer-vnet-subnet-candidate-tv4mch\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-40/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"ab4d77db9ca57b16c89060879e2cdf736f327816be7ca7e7c5a11ef73809df8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"b5c012efbb5a84dfc1c92d906336f3fd471a9307246feb608c1c3390ba7e841e","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-8klpvf\" with version 5.0.0-rc.3 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009803c0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-8klpvf: failed waiting for cluster=\"cluster-zstream-5-0-8klpvf\" in resourcegroup=\"rg-zstream-upgrade-5-0-5q8zb7dl26ph\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/842e7207-d3c7-4e3f-be1a-57ee19c67a21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/842e7207-d3c7-4e3f-be1a-57ee19c67a21\",\n \"name\": \"842e7207-d3c7-4e3f-be1a-57ee19c67a21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:02.432476672Z\",\n \"endTime\": \"2026-09-29T10:13:22.054359416Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-5q8zb7dl26ph/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-8klpvf\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-5q8zb7dl26ph/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-8klpvf\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-5q8zb7dl26ph/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-8klpvf/subnets/customer-vnet-subnet-zstream-8klpvf\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-8klpvf\" with version 5.0.0-rc.3 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009803c0\u003e: \n failed to create HCP cluster cluster-zstream-5-0-8klpvf: failed waiting for cluster=\"cluster-zstream-5-0-8klpvf\" in resourcegroup=\"rg-zstream-upgrade-5-0-5q8zb7dl26ph\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/842e7207-d3c7-4e3f-be1a-57ee19c67a21\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/842e7207-d3c7-4e3f-be1a-57ee19c67a21\",\n \"name\": \"842e7207-d3c7-4e3f-be1a-57ee19c67a21\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:02.432476672Z\",\n \"endTime\": \"2026-09-29T10:13:22.054359416Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-5q8zb7dl26ph/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-8klpvf\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-5q8zb7dl26ph/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-8klpvf\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-5-0-5q8zb7dl26ph/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-8klpvf/subnets/customer-vnet-subnet-zstream-8klpvf\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"b3e5abef314ee54c0d8e226334a28bc9d8c569583cc7054ce31753798fa77e9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"alert","signature_id":"3b8237c385c679cef0fb27bf817f6b03b5a4d1679095547dd07743b8feb9c91f","test_name":"[aro-hcp-observability] [svc] alert userJourneyClusterProvisionErrors1h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-29T10:20:17Z\n Severity: Sev3\n Labels: alertname=\"userJourneyClusterProvisionErrors1h5m\", cluster=\"ci01-j1689472-svc\", component=\"slo\", long_window=\"1h\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1689472/providers/microsoft.monitor/accounts/services-j1689472\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"cluster-provision-errors\"\n Description: More than 72% of completed cluster create (install) operations on ci01-j1689472-svc failed over the last hour with at least 3 failures, a 14.4x burn of the 95% SLO budget.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","failure_pattern":"alert [svc] userJourneyClusterProvisionErrors1h5m fired","search_query":"alert [svc] userJourneyClusterProvisionErrors1h5m fired"}},{"row_id":"b88dec3eca2e3840da0462bebbf5401777071a34abe639592629ea6780ecc623","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"026f4166fb5ca952838cfd2e97e9bdb29a82c767f2354335acdf20b443285364","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hzw8jglf845v/cluster-candidate-4-21-779mcz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008bc800\u003e: \n failed to create HCP cluster cluster-candidate-4-21-779mcz: failed waiting for cluster=\"cluster-candidate-4-21-779mcz\" in resourcegroup=\"rg-candidate-4-21-hzw8jglf845v\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/650e9430-8570-4b8f-b095-fa3e61539c0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/650e9430-8570-4b8f-b095-fa3e61539c0e\",\n \"name\": \"650e9430-8570-4b8f-b095-fa3e61539c0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:50.884256802Z\",\n \"endTime\": \"2026-09-29T10:12:21.595783509Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hzw8jglf845v/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-779mcz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hzw8jglf845v/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-779mcz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hzw8jglf845v/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-779mcz/subnets/customer-vnet-subnet-candidate-779mcz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-hzw8jglf845v/cluster-candidate-4-21-779mcz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008bc800\u003e: \n failed to create HCP cluster cluster-candidate-4-21-779mcz: failed waiting for cluster=\"cluster-candidate-4-21-779mcz\" in resourcegroup=\"rg-candidate-4-21-hzw8jglf845v\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/650e9430-8570-4b8f-b095-fa3e61539c0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/650e9430-8570-4b8f-b095-fa3e61539c0e\",\n \"name\": \"650e9430-8570-4b8f-b095-fa3e61539c0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:50.884256802Z\",\n \"endTime\": \"2026-09-29T10:12:21.595783509Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hzw8jglf845v/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-candidate-779mcz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hzw8jglf845v/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-779mcz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hzw8jglf845v/providers/Microsoft.Network/virtualNetworks/customer-vnet-candidate-779mcz/subnets/customer-vnet-subnet-candidate-779mcz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-30/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"b970771a42d90eca4243108380ad53dfb82c59d21daaf2feed8e13f29e17f299","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"a6c78cb691164be897ebf2e98976dc48e6379f9526a414874ff1549cd9656cf9","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000595020\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-qtksnmktlq44\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a8d6ac3-d431-472f-a0bb-f5d012f1439c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a8d6ac3-d431-472f-a0bb-f5d012f1439c\",\n \"name\": \"3a8d6ac3-d431-472f-a0bb-f5d012f1439c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:21.586028068Z\",\n \"endTime\": \"2026-09-29T10:16:11.26719174Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-qtksnmktlq44/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-qtksnmktlq44/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-qtksnmktlq44/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet/subnets/arm64-vm-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000595020\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-qtksnmktlq44\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a8d6ac3-d431-472f-a0bb-f5d012f1439c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a8d6ac3-d431-472f-a0bb-f5d012f1439c\",\n \"name\": \"3a8d6ac3-d431-472f-a0bb-f5d012f1439c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:21.586028068Z\",\n \"endTime\": \"2026-09-29T10:16:11.26719174Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-qtksnmktlq44/providers/Microsoft.Network/networkSecurityGroups/arm64-vm-customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-qtksnmktlq44/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/arm64-vm-cluster-qtksnmktlq44/providers/Microsoft.Network/virtualNetworks/arm64-vm-customer-vnet/subnets/arm64-vm-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-02/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"bd36875ad49dec9b0a10abb083f91d23dd77c5830728f9a636ef613676724d4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"04afd3e556de1df9ea78271cb1e11b46ed0c7bf1e77d123b868ac4998ddf7999","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-cxd8l4\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00123e500\u003e: \n failed to create HCP cluster np-noedge-cxd8l4: failed waiting for cluster=\"np-noedge-cxd8l4\" in resourcegroup=\"rg-np-noedge-cxd8l4-h9t6xzrtxdvv\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b33b6ae-0be6-4497-a86f-67ea2a4de210\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b33b6ae-0be6-4497-a86f-67ea2a4de210\",\n \"name\": \"3b33b6ae-0be6-4497-a86f-67ea2a4de210\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:26:32.054517719Z\",\n \"endTime\": \"2026-09-29T10:37:32.806465773Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-cxd8l4-h9t6xzrtxdvv/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-noedge-cxd8l4\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-cxd8l4-h9t6xzrtxdvv/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-noedge-cxd8l4\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-cxd8l4-h9t6xzrtxdvv/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-noedge-cxd8l4/subnets/customer-vnet-subnet-np-noedge-cxd8l4\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-cxd8l4\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00123e500\u003e: \n failed to create HCP cluster np-noedge-cxd8l4: failed waiting for cluster=\"np-noedge-cxd8l4\" in resourcegroup=\"rg-np-noedge-cxd8l4-h9t6xzrtxdvv\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b33b6ae-0be6-4497-a86f-67ea2a4de210\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b33b6ae-0be6-4497-a86f-67ea2a4de210\",\n \"name\": \"3b33b6ae-0be6-4497-a86f-67ea2a4de210\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:26:32.054517719Z\",\n \"endTime\": \"2026-09-29T10:37:32.806465773Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-cxd8l4-h9t6xzrtxdvv/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-noedge-cxd8l4\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-cxd8l4-h9t6xzrtxdvv/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-noedge-cxd8l4\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-cxd8l4-h9t6xzrtxdvv/providers/Microsoft.Network/virtualNetworks/customer-vnet-np-noedge-cxd8l4/subnets/customer-vnet-subnet-np-noedge-cxd8l4\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-37/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"c13dc82781ebe1a8de226f7b263964cb7e9596c32220817af8efd8cb8070de1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"a56da4eb8a1104029704d6a6534734f044a8a33af05589281b251afcbbb9c77b","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106a5c0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-8s87zqr6fqc8\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3825318e-0b9f-4211-af4e-141a281be97c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3825318e-0b9f-4211-af4e-141a281be97c\",\n \"name\": \"3825318e-0b9f-4211-af4e-141a281be97c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:44.573548584Z\",\n \"endTime\": \"2026-09-29T10:14:01.317741909Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-8s87zqr6fqc8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-8s87zqr6fqc8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-8s87zqr6fqc8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:113]: failed to create HCP cluster for pull secret test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106a5c0\u003e: \n failed to create HCP cluster pullsecret-hcp-cluster: failed waiting for cluster=\"pullsecret-hcp-cluster\" in resourcegroup=\"pullsecret-test-8s87zqr6fqc8\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3825318e-0b9f-4211-af4e-141a281be97c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3825318e-0b9f-4211-af4e-141a281be97c\",\n \"name\": \"3825318e-0b9f-4211-af4e-141a281be97c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:44.573548584Z\",\n \"endTime\": \"2026-09-29T10:14:01.317741909Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-8s87zqr6fqc8/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-8s87zqr6fqc8/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/pullsecret-test-8s87zqr6fqc8/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-18/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"c705090382a8f0ebb00a6fe39de4cab5204befdbca4f69550481fb9e61b645e3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"2e2523804d98de3b19b0687bdd6f57b081097790b646284bfebad965a99eb3be","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df6ac0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-jv8gmddt7jdv\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9db80261-68e0-4700-9603-e76b68300133\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9db80261-68e0-4700-9603-e76b68300133\",\n \"name\": \"9db80261-68e0-4700-9603-e76b68300133\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:14.50722231Z\",\n \"endTime\": \"2026-09-29T10:14:11.459675493Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-jv8gmddt7jdv/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-59/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-jv8gmddt7jdv/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-59/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-jv8gmddt7jdv/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-59/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df6ac0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-jv8gmddt7jdv\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9db80261-68e0-4700-9603-e76b68300133\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9db80261-68e0-4700-9603-e76b68300133\",\n \"name\": \"9db80261-68e0-4700-9603-e76b68300133\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:14.50722231Z\",\n \"endTime\": \"2026-09-29T10:14:11.459675493Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-jv8gmddt7jdv/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-59/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-jv8gmddt7jdv/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-59/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-update-nodes-jv8gmddt7jdv/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-59/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"d182f6070136fe2011f4473cde5b78ef3fd98acaa7fe7e1225a1f17eb0d642fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"72d3d2f5a1468eed26edd84846b87b653a72ce4389b93df4e7eb2b5cdd7906d8","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1a320\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jtlkrx8rzb5x\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6f04b88-001b-4147-942d-d885ef16e9ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6f04b88-001b-4147-942d-d885ef16e9ed\",\n \"name\": \"e6f04b88-001b-4147-942d-d885ef16e9ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:01:22.5908221Z\",\n \"endTime\": \"2026-09-29T10:15:41.462886265Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-jtlkrx8rzb5x/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-jtlkrx8rzb5x/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-jtlkrx8rzb5x/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b1a320\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jtlkrx8rzb5x\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6f04b88-001b-4147-942d-d885ef16e9ed\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e6f04b88-001b-4147-942d-d885ef16e9ed\",\n \"name\": \"e6f04b88-001b-4147-942d-d885ef16e9ed\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:01:22.5908221Z\",\n \"endTime\": \"2026-09-29T10:15:41.462886265Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-jtlkrx8rzb5x/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-jtlkrx8rzb5x/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-jtlkrx8rzb5x/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-12/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"dcaf0a1410f186ad63fee78ab20dc587cd3f65944100c77781e07531dff36ea5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"a6947297c3e0bb4a3a2f33654efa3d2c54df093292de44470164859158f981e7","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e88b40\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-l99z289gbk46\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15548681-6851-4a12-9b45-286900a99a28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15548681-6851-4a12-9b45-286900a99a28\",\n \"name\": \"15548681-6851-4a12-9b45-286900a99a28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:50.300559846Z\",\n \"endTime\": \"2026-09-29T10:12:54.771194534Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-l99z289gbk46/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-l99z289gbk46/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-l99z289gbk46/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e88b40\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-l99z289gbk46\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15548681-6851-4a12-9b45-286900a99a28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15548681-6851-4a12-9b45-286900a99a28\",\n \"name\": \"15548681-6851-4a12-9b45-286900a99a28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:50.300559846Z\",\n \"endTime\": \"2026-09-29T10:12:54.771194534Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-l99z289gbk46/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-l99z289gbk46/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-cluster-nsg-subnet-reuse-l99z289gbk46/providers/Microsoft.Network/virtualNetworks/customer-vnet-name1/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-03/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"dcf11b86e293dfe32d9b885ae73ab756550395a5b6ad93ae2c95862b8a2e2cd9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"3e66c5b00d6c9ceaac1d6622926c99e79a7b0ea1060e7540c8ee6cc7761ee7bb","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000544480\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-v7tg2pl2vssw\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3513cb3c-542a-45e2-b890-b74e5ca8df6c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3513cb3c-542a-45e2-b890-b74e5ca8df6c\",\n \"name\": \"3513cb3c-542a-45e2-b890-b74e5ca8df6c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.11722386Z\",\n \"endTime\": \"2026-09-29T10:13:31.303503246Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-v7tg2pl2vssw/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-10/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-v7tg2pl2vssw/providers/Microsoft.Network/virtualNetworks/sre-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-10/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-v7tg2pl2vssw/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-10/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000544480\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-v7tg2pl2vssw\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3513cb3c-542a-45e2-b890-b74e5ca8df6c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3513cb3c-542a-45e2-b890-b74e5ca8df6c\",\n \"name\": \"3513cb3c-542a-45e2-b890-b74e5ca8df6c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:11.11722386Z\",\n \"endTime\": \"2026-09-29T10:13:31.303503246Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-v7tg2pl2vssw/providers/Microsoft.Network/networkSecurityGroups/sre-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-10/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-v7tg2pl2vssw/providers/Microsoft.Network/virtualNetworks/sre-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-10/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-api-breakglass-v7tg2pl2vssw/providers/Microsoft.Network/virtualNetworks/sre-vnet-name/subnets/sre-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-10/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"df975aad71a059170753c64ead202b9d82ea1ac1f6957ee052fc09f2ed7bc84b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"473822b3bb02c121178d54feab4456fe8182b77fed856c20c137174979cc352c","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-f25ftz\" with version 4.22.15 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00030c740\u003e: \n failed to create HCP cluster cluster-zstream-4-22-f25ftz: failed waiting for cluster=\"cluster-zstream-4-22-f25ftz\" in resourcegroup=\"rg-zstream-upgrade-4-22-6c5hb4q5cxc9\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29db6dda-3b86-4e79-a7f8-37c69a877297\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29db6dda-3b86-4e79-a7f8-37c69a877297\",\n \"name\": \"29db6dda-3b86-4e79-a7f8-37c69a877297\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:50.726286002Z\",\n \"endTime\": \"2026-09-29T10:15:31.64753665Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-6c5hb4q5cxc9/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-f25ftz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-6c5hb4q5cxc9/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-f25ftz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-6c5hb4q5cxc9/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-f25ftz/subnets/customer-vnet-subnet-zstream-f25ftz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-f25ftz\" with version 4.22.15 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00030c740\u003e: \n failed to create HCP cluster cluster-zstream-4-22-f25ftz: failed waiting for cluster=\"cluster-zstream-4-22-f25ftz\" in resourcegroup=\"rg-zstream-upgrade-4-22-6c5hb4q5cxc9\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29db6dda-3b86-4e79-a7f8-37c69a877297\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/29db6dda-3b86-4e79-a7f8-37c69a877297\",\n \"name\": \"29db6dda-3b86-4e79-a7f8-37c69a877297\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:50.726286002Z\",\n \"endTime\": \"2026-09-29T10:15:31.64753665Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-6c5hb4q5cxc9/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-zstream-f25ftz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-6c5hb4q5cxc9/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-f25ftz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-22-6c5hb4q5cxc9/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-f25ftz/subnets/customer-vnet-subnet-zstream-f25ftz\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-33/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"eb0fef81b2a3213fedad2e0e4d74e84bf70b5feae4a9476d7185f13de53e245c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"aa5193a9e49b32f4a6fb8a9acb7df11be590178550405f12c93db0516a804f5e","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089cda0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4f9dz7cnqptl\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8300703-4064-4289-9b8a-058b057e131e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8300703-4064-4289-9b8a-058b057e131e\",\n \"name\": \"d8300703-4064-4289-9b8a-058b057e131e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:18.422722674Z\",\n \"endTime\": \"2026-09-29T10:15:52.853733922Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4f9dz7cnqptl/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4f9dz7cnqptl/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4f9dz7cnqptl/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00089cda0\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-4f9dz7cnqptl\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8300703-4064-4289-9b8a-058b057e131e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d8300703-4064-4289-9b8a-058b057e131e\",\n \"name\": \"d8300703-4064-4289-9b8a-058b057e131e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:18.422722674Z\",\n \"endTime\": \"2026-09-29T10:15:52.853733922Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4f9dz7cnqptl/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4f9dz7cnqptl/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-4f9dz7cnqptl/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-53/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"eda74abec0a6f4e650a7d302e7366223afcbf77143c24d420e956a1d6f6218a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"9d365b794b110ccc7d38e09b1a3a34705acae007cc3f672c8781af51a6c3de8a","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000998a00\u003e: \n failed to create HCP cluster ea-listgw94mn: failed waiting for cluster=\"ea-listgw94mn\" in resourcegroup=\"ea-list-d7zlwmdq2v8p\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e821ea6-3475-4962-ac2d-88b29f66e816\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e821ea6-3475-4962-ac2d-88b29f66e816\",\n \"name\": \"1e821ea6-3475-4962-ac2d-88b29f66e816\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:35.610914879Z\",\n \"endTime\": \"2026-09-29T10:16:31.031885218Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-d7zlwmdq2v8p/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-d7zlwmdq2v8p/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-d7zlwmdq2v8p/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000998a00\u003e: \n failed to create HCP cluster ea-listgw94mn: failed waiting for cluster=\"ea-listgw94mn\" in resourcegroup=\"ea-list-d7zlwmdq2v8p\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e821ea6-3475-4962-ac2d-88b29f66e816\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e821ea6-3475-4962-ac2d-88b29f66e816\",\n \"name\": \"1e821ea6-3475-4962-ac2d-88b29f66e816\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:35.610914879Z\",\n \"endTime\": \"2026-09-29T10:16:31.031885218Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-d7zlwmdq2v8p/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-d7zlwmdq2v8p/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-d7zlwmdq2v8p/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-47/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"f2113b9fa095104f90f3b8bc72e92b511eab4f771cfcd998819cc310e0da8985","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"b16ff26a1cf0fec6d57da260d65a7621be628c299762aceb5b0a1021aca849c2","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006e8f20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-js2sxjs464qq\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1f42a42-f15b-4b81-9202-9323242c990b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1f42a42-f15b-4b81-9202-9323242c990b\",\n \"name\": \"a1f42a42-f15b-4b81-9202-9323242c990b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:36.823562108Z\",\n \"endTime\": \"2026-09-29T10:16:31.034572694Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-js2sxjs464qq/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-js2sxjs464qq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-js2sxjs464qq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006e8f20\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-js2sxjs464qq\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1f42a42-f15b-4b81-9202-9323242c990b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a1f42a42-f15b-4b81-9202-9323242c990b\",\n \"name\": \"a1f42a42-f15b-4b81-9202-9323242c990b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:36.823562108Z\",\n \"endTime\": \"2026-09-29T10:16:31.034572694Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-js2sxjs464qq/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-js2sxjs464qq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-js2sxjs464qq/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-48/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"f26ad82508c64be7ddcb88483e53c6c5c6d0d5225ed75eb58f0e4882d9b98379","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"e9ac3f4e2fc49171a6b30364a8698157781411b5cb618ad96abb5ec0e4448054","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd0860\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tz6pltrbbc9x\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7855d105-d623-41db-8d0d-5cf29ee8ddba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7855d105-d623-41db-8d0d-5cf29ee8ddba\",\n \"name\": \"7855d105-d623-41db-8d0d-5cf29ee8ddba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:01:04.886386488Z\",\n \"endTime\": \"2026-09-29T10:17:31.141351258Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tz6pltrbbc9x/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tz6pltrbbc9x/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tz6pltrbbc9x/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd0860\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-tz6pltrbbc9x\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7855d105-d623-41db-8d0d-5cf29ee8ddba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7855d105-d623-41db-8d0d-5cf29ee8ddba\",\n \"name\": \"7855d105-d623-41db-8d0d-5cf29ee8ddba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:01:04.886386488Z\",\n \"endTime\": \"2026-09-29T10:17:31.141351258Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tz6pltrbbc9x/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tz6pltrbbc9x/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-tz6pltrbbc9x/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-23/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"f3df5d1c0129acfb086a4c2de7429787b22ff7b243bb6a202991466a1a365cae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"00920d7a96bfc4dc834914344c9d0d91259e4ca574824e69ce294f8f855eb6e5","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010386e0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-7vndfwh8dc5z\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c0cfb255-b550-40af-8f39-679f917ffd5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c0cfb255-b550-40af-8f39-679f917ffd5f\",\n \"name\": \"c0cfb255-b550-40af-8f39-679f917ffd5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:52.42060332Z\",\n \"endTime\": \"2026-09-29T10:15:11.349723956Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-7vndfwh8dc5z/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-58/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-7vndfwh8dc5z/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-58/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-7vndfwh8dc5z/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-58/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010386e0\u003e: \n failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-7vndfwh8dc5z\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c0cfb255-b550-40af-8f39-679f917ffd5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c0cfb255-b550-40af-8f39-679f917ffd5f\",\n \"name\": \"c0cfb255-b550-40af-8f39-679f917ffd5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:52.42060332Z\",\n \"endTime\": \"2026-09-29T10:15:11.349723956Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-7vndfwh8dc5z/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-58/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-7vndfwh8dc5z/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-58/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-7vndfwh8dc5z/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-58/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"f5c2447293a1e8ed150e5b465effd70509d729e242b453a11fec2bb37626bb99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"fe206f80842cfdb2d276311b774e61bef20a0b87d5b320b5a3cc0cbb62316923","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001208380\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-l2s2cdz29mxp\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c20ae6ea-baa3-4eda-b7f3-01759c82f1fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c20ae6ea-baa3-4eda-b7f3-01759c82f1fa\",\n \"name\": \"c20ae6ea-baa3-4eda-b7f3-01759c82f1fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:27:52.770463042Z\",\n \"endTime\": \"2026-09-29T10:38:32.770505796Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l2s2cdz29mxp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l2s2cdz29mxp/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l2s2cdz29mxp/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001208380\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-l2s2cdz29mxp\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c20ae6ea-baa3-4eda-b7f3-01759c82f1fa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c20ae6ea-baa3-4eda-b7f3-01759c82f1fa\",\n \"name\": \"c20ae6ea-baa3-4eda-b7f3-01759c82f1fa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:27:52.770463042Z\",\n \"endTime\": \"2026-09-29T10:38:32.770505796Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l2s2cdz29mxp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l2s2cdz29mxp/providers/Microsoft.Network/virtualNetworks/customer-vnet-name\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-negative-tests-l2s2cdz29mxp/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/subnets/customer-vnet-subnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"f78e3b66e960827a825f39240690538455a56b19abe4c5781d060d0b8d63e466","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"a56747c048aa13def2ea5f2bc32b68d41ae1426a61709b237c6ea023b265ebf0","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d11a0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cwbljmg5wx8g\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899f3956-3607-48cb-9d16-2c67f9649df1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899f3956-3607-48cb-9d16-2c67f9649df1\",\n \"name\": \"899f3956-3607-48cb-9d16-2c67f9649df1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:50.553139534Z\",\n \"endTime\": \"2026-09-29T10:14:21.47198569Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cwbljmg5wx8g/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cwbljmg5wx8g/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cwbljmg5wx8g/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d11a0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-cwbljmg5wx8g\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899f3956-3607-48cb-9d16-2c67f9649df1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/899f3956-3607-48cb-9d16-2c67f9649df1\",\n \"name\": \"899f3956-3607-48cb-9d16-2c67f9649df1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:59:50.553139534Z\",\n \"endTime\": \"2026-09-29T10:14:21.47198569Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cwbljmg5wx8g/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cwbljmg5wx8g/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-cwbljmg5wx8g/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"fc6f9e9fa723eb4fdaaa0c4a7cf81c7987c3d352ac8a1d3a553d165c62f818b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"add775b98a987bfc6ef07dabdb605badeffea9df24cd04a7e07533710f867137","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a92160\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-zlbhbdc44g5f\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/05a8ba70-8fca-4c1d-a2c7-d68db104a4c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/05a8ba70-8fca-4c1d-a2c7-d68db104a4c3\",\n \"name\": \"05a8ba70-8fca-4c1d-a2c7-d68db104a4c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:01:42.527610475Z\",\n \"endTime\": \"2026-09-29T10:14:01.252062278Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zlbhbdc44g5f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zlbhbdc44g5f/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zlbhbdc44g5f/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a92160\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-zlbhbdc44g5f\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/05a8ba70-8fca-4c1d-a2c7-d68db104a4c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/05a8ba70-8fca-4c1d-a2c7-d68db104a4c3\",\n \"name\": \"05a8ba70-8fca-4c1d-a2c7-d68db104a4c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:01:42.527610475Z\",\n \"endTime\": \"2026-09-29T10:14:01.252062278Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zlbhbdc44g5f/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zlbhbdc44g5f/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zlbhbdc44g5f/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-17/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"fea300f7880324cb47473b2578d3c2c4a4a544e41a09464850596bffc0933328","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"b4d1596a0f70e533af4c38640fccaed6d908968ddb0f9026c61175bd677da09b","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001052600\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-bt6zt7rg5snt\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1857a7-ef23-448a-9b13-a4d739031ad9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1857a7-ef23-448a-9b13-a4d739031ad9\",\n \"name\": \"8e1857a7-ef23-448a-9b13-a4d739031ad9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:01.692868582Z\",\n \"endTime\": \"2026-09-29T10:13:31.119126573Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-bt6zt7rg5snt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-bt6zt7rg5snt/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-bt6zt7rg5snt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:86]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001052600\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-bt6zt7rg5snt\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1857a7-ef23-448a-9b13-a4d739031ad9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8e1857a7-ef23-448a-9b13-a4d739031ad9\",\n \"name\": \"8e1857a7-ef23-448a-9b13-a4d739031ad9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:01.692868582Z\",\n \"endTime\": \"2026-09-29T10:13:31.119126573Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-bt6zt7rg5snt/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-bt6zt7rg5snt/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-bt6zt7rg5snt/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-20/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}},{"row_id":"ffeca13ab4587c450ddad752392f124f6a28675a9ff3d6892b3b8c38a5f80b6e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104860015251689472","occurred_at":"2026-09-29T09:05:08Z","failed_at":"e2e","signature_id":"6edcd76d2e80e758a8a41a46e005536daf433a34adc1163102a9573e96c8657a","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cea1a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-x7hdzd8gx6dk\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db88ce37-7c62-4edc-9299-c2d56f61df87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db88ce37-7c62-4edc-9299-c2d56f61df87\",\n \"name\": \"db88ce37-7c62-4edc-9299-c2d56f61df87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:40.366599036Z\",\n \"endTime\": \"2026-09-29T10:17:52.077450692Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-x7hdzd8gx6dk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-x7hdzd8gx6dk/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-x7hdzd8gx6dk/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cea1a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-x7hdzd8gx6dk\" to finish creating: GET https://rp.j1689472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db88ce37-7c62-4edc-9299-c2d56f61df87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InvalidResource\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/db88ce37-7c62-4edc-9299-c2d56f61df87\",\n \"name\": \"db88ce37-7c62-4edc-9299-c2d56f61df87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:00:40.366599036Z\",\n \"endTime\": \"2026-09-29T10:17:52.077450692Z\",\n \"error\": {\n \"code\": \"InvalidResource\",\n \"message\": \"DataPlaneIdentitiesPermissionsValidation: MissingRequiredPermissions: Data plane operators missing required permissions: [{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-x7hdzd8gx6dk/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/networkSecurityGroups/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-x7hdzd8gx6dk/providers/Microsoft.Network/virtualNetworks/customer-vnet\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]},{\\\"resource\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-x7hdzd8gx6dk/providers/Microsoft.Network/virtualNetworks/customer-vnet/subnets/customer-subnet-1\\\",\\\"identity\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard1-00-56/providers/Microsoft.ManagedIdentity/userAssignedIdentities/dp-file-csi-driver\\\",\\\"decisions\\\":[{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/read\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/write\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"},{\\\"actionId\\\":\\\"Microsoft.Network/virtualNetworks/subnets/join/action\\\",\\\"isDataAction\\\":false,\\\"accessDecision\\\":\\\"NotAllowed\\\"}]}]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","failure_pattern":"ERROR CODE: InvalidResource; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: InvalidResource"}}],"failure_pattern_summary":{"signature_count":67,"failure_pattern_ids":["1718f77cda36f34fdeeb83e80420de5016b278e60ddf07d8f276d1ddaa71ee55","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","9a8854d173302655314e0bdcbc8411f9e276ffe0aadf4e25f0e0bad8f8dcb51d","d11e817743c9b7b223c4f7499ea10417bad7afb08d112eac38001716535808b0","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":67,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104858665063288832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T08:59:46Z","started_at":"2026-09-29T08:59:46Z","completed_at":"2026-09-29T11:15:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T11:26:35.783010722Z","timing_metadata_checked_at":"2026-09-29T11:26:35.783010722Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T11:26:35.794557378Z","region_metadata_checked_at":"2026-09-29T11:26:35.794557378Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dc31a106abf4a7cdfbd2feaf179f9757983024c59e5abac884c620dd06997305","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104858665063288832","occurred_at":"2026-09-29T08:59:46Z","failed_at":"e2e","signature_id":"80350b99fd456972b62772689ba44d31bac1262ed819b93e1ae298e0c40f49aa","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0015823f0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0015823f0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7236/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104856376063823872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7236,"pr_state":"closed","pr_sha":"05c4e5660899625033dcf08a3920a36abe3c5b67","final_merged_sha":"d5cf9a520986ae23608f3ac67830e681c126c99c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T08:50:39Z","started_at":"2026-09-29T08:50:39Z","completed_at":"2026-09-29T11:10:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T11:26:35.975014488Z","timing_metadata_checked_at":"2026-09-29T11:26:35.975014488Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T11:26:35.978216476Z","region_metadata_checked_at":"2026-09-29T11:26:35.978216476Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"240d114d373a7b6f98d3e1e7da1ac92379b94ebbceb51b2819efc0cccef26714","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7236/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104856376063823872","occurred_at":"2026-09-29T08:50:39Z","failed_at":"e2e","signature_id":"22b8b5d5909cf69ee83879be013f44ecfb61e440816377ec7e0c466ad8e1d467","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000cd6168\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000cd6168\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"583ea0ee6293cef7bfe31099412fb1e4f700a44780c238c88d6ab1fa219e0b30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7236/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104856376063823872","occurred_at":"2026-09-29T08:50:39Z","failed_at":"e2e","signature_id":"4974bcdffae3f60c4b9972d14a15f1943747c1fa6bf579aaf54d37821dd4c418","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103c100\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-b47p4t569j8q\" to finish creating: GET https://rp.j3823872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b27fd82-0e24-4270-83d4-2e9185573f24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b27fd82-0e24-4270-83d4-2e9185573f24\",\n \"name\": \"9b27fd82-0e24-4270-83d4-2e9185573f24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:47:58.881966525Z\",\n \"endTime\": \"2026-09-29T10:17:01.247657057Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103c100\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-b47p4t569j8q\" to finish creating: GET https://rp.j3823872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b27fd82-0e24-4270-83d4-2e9185573f24\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9b27fd82-0e24-4270-83d4-2e9185573f24\",\n \"name\": \"9b27fd82-0e24-4270-83d4-2e9185573f24\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T09:47:58.881966525Z\",\n \"endTime\": \"2026-09-29T10:17:01.247657057Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9457ca41aa4fe3b1aaa2c082141152f089bb940c3909c02cc5ef2a4af627af8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7236/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104856376063823872","occurred_at":"2026-09-29T08:50:39Z","failed_at":"e2e","signature_id":"19f4977f53c63c72dfde2e6438460984bffea5126f3480fe4162e03cd88d6837","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d06b20\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-gppxxf\" in resourcegroup=\"rg-np-version-upgrade-gppxxf-t8pfflgzg62v\" to finish creating: GET https://rp.j3823872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd03c1a-8c8a-491e-884b-41c3a483beaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd03c1a-8c8a-491e-884b-41c3a483beaa\",\n \"name\": \"8dd03c1a-8c8a-491e-884b-41c3a483beaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:16:33.80832542Z\",\n \"endTime\": \"2026-09-29T10:35:41.266382183Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o1n3u4f1r1o4s1b-npupgrade-4-20-wkxp9-vwkb9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-gppxxf-t8pfflgzg62v-np-upgrade-gp-c3842a03/providers/Microsoft.Compute/virtualMachines/o1n3u4f1r1o4s1b-npupgrade-4-20-wkxp9-vwkb9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o1n3u4f1r1o4s1b-npupgrade-4-20-wkxp9-vwkb9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-gppxxf-t8pfflgzg62v-np-upgrade-gp-c3842a03/providers/Microsoft.Compute/virtualMachines/o1n3u4f1r1o4s1b-npupgrade-4-20-wkxp9-vwkb9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d06b20\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-gppxxf\" in resourcegroup=\"rg-np-version-upgrade-gppxxf-t8pfflgzg62v\" to finish creating: GET https://rp.j3823872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd03c1a-8c8a-491e-884b-41c3a483beaa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dd03c1a-8c8a-491e-884b-41c3a483beaa\",\n \"name\": \"8dd03c1a-8c8a-491e-884b-41c3a483beaa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T10:16:33.80832542Z\",\n \"endTime\": \"2026-09-29T10:35:41.266382183Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine o1n3u4f1r1o4s1b-npupgrade-4-20-wkxp9-vwkb9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-gppxxf-t8pfflgzg62v-np-upgrade-gp-c3842a03/providers/Microsoft.Compute/virtualMachines/o1n3u4f1r1o4s1b-npupgrade-4-20-wkxp9-vwkb9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine o1n3u4f1r1o4s1b-npupgrade-4-20-wkxp9-vwkb9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-gppxxf-t8pfflgzg62v-np-upgrade-gp-c3842a03/providers/Microsoft.Compute/virtualMachines/o1n3u4f1r1o4s1b-npupgrade-4-20-wkxp9-vwkb9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"df700d4672957b7934bc6261e12ef21719d7b6fb9040556cd95cf202897699a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7236/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104856376063823872","occurred_at":"2026-09-29T08:50:39Z","failed_at":"alert","signature_id":"5e111eac7ea24c2ce31a7286a9c38f9ca892d7ac60be751f43496321f784b6c4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T09:58:08Z\n Ended: 2026-09-29T10:29:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3823872-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4jmk564mnm12sgb8v1n0pugtka318g-cilium-cluster\", pod=\"router-7c9c8d96c5-5t9sh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4jmk564mnm12sgb8v1n0pugtka318g-cilium-cluster/router-7c9c8d96c5-5t9sh has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","36d5ba2573b75da6f6f0744a2a3f59859305e1069fd3c0de873f2ce3fccad023","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104846989291163648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7071,"pr_state":"open","pr_sha":"a3171eba91feacfd5f8fe62df0a9912b7651fcc1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T08:13:23Z","started_at":"2026-09-29T08:13:23Z","completed_at":"2026-09-29T08:47:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:00:36.751099404Z","timing_metadata_checked_at":"2026-09-29T09:00:36.751099404Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:00:36.754328294Z","region_metadata_checked_at":"2026-09-29T09:00:36.754328294Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b264b0c17c05121a52655e2ee8dd76622cb50a506ec7d41135b140fc74f1c3a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104846989291163648","occurred_at":"2026-09-29T08:13:23Z","failed_at":"provision","signature_id":"cf1f597a724be7b06b212cc3c9376e4ade9bf38f4eb4383b5e5df9663f45bfc4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-29T08:34:25.705Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-29T08:42:03.922Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-29T08:34:25.714664724Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-29T08:34:27.934335048Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-29T08:34:28.153595232Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T08:42:03.919377937Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E0CC533F465FCB4C\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1163648-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j1163648-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T08:39:20.5956034Z\\\",\\\"duration\\\":\\\"PT1M53.0427309S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T08:42:03.919443378Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"21FA5A5958E17E85\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1163648-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j1163648-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T08:39:20.5956034Z\\\",\\\"duration\\\":\\\"PT1M37.3664627S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T08:42:03.919464379Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"3FBD33CFC93FD8AF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1163648-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j1163648-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-29T08:39:20.5956034Z\\\",\\\"duration\\\":\\\"PT35.412467S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-12237352-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-12237352-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-29T08:42:03.919563681Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"F8F7C66DAE59E102\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1163648-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j1163648-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T08:39:20.5956034Z\\\",\\\"duration\\\":\\\"PT1M22.2906097S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T08:42:03.919594142Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"109FB9AA097998DF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1163648-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j1163648-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T08:34:33.6987466Z\\\",\\\"duration\\\":\\\"PT7M26.7702251S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T08:42:03.919629593Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A22B902E63DA51E7\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1163648-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T08:34:33.6987466Z\\\",\\\"duration\\\":\\\"PT4M46.8300264S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T08:42:03.919662274Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j1163648-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j1163648-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1163648-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109360170854697\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j1163648-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-12237352-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1163648-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-12237352-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m38.20500018s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7236/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104829939730616320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7236,"pr_state":"closed","pr_sha":"05c4e5660899625033dcf08a3920a36abe3c5b67","final_merged_sha":"d5cf9a520986ae23608f3ac67830e681c126c99c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T07:05:38Z","started_at":"2026-09-29T07:05:38Z","completed_at":"2026-09-29T07:42:50Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T07:46:14.020845045Z","timing_metadata_checked_at":"2026-09-29T07:46:14.020845045Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T07:46:14.023450138Z","region_metadata_checked_at":"2026-09-29T07:46:14.023450138Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"89d2e93432e62788b4198ca03838f6004c9d1f2879a00e25791a3d12700c232c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7236/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104829939730616320","occurred_at":"2026-09-29T07:05:38Z","failed_at":"provision","signature_id":"2464bf8c03131eec64bb75bb40d915e697ae2a30c51fea5179d180a8127496a7","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-29T07:29:07.654Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-29T07:37:16.632Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-29T07:29:07.663748341Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-29T07:29:09.664580159Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-29T07:29:09.774367851Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:37:16.628236265Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E60E8144CEBBD0EE\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0616320-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j0616320-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:34:33.4549643Z\\\",\\\"duration\\\":\\\"PT1M54.5572403S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:37:16.628296067Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"3B57D2CEB495CF19\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0616320-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j0616320-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-29T07:34:33.4549643Z\\\",\\\"duration\\\":\\\"PT1M22.0705191S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"3\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41550067-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41550067-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-29T07:37:16.628368178Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1FDC71FA470AB50C\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0616320-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j0616320-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:29:15.5482717Z\\\",\\\"duration\\\":\\\"PT6M40.5743573S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:37:16.628379928Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E9836E8873033651\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0616320-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j0616320-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:34:33.4549643Z\\\",\\\"duration\\\":\\\"PT1M38.3097269S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:37:16.628390179Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"419B3E4C9E087C8E\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0616320-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j0616320-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:34:33.4549643Z\\\",\\\"duration\\\":\\\"PT2M23.5295769S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:37:16.628401639Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"382BA92BCBB89114\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0616320-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:29:15.5482717Z\\\",\\\"duration\\\":\\\"PT5M17.7846591S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:37:16.628426019Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0616320-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j0616320-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0616320-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109399346000714\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j0616320-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41550067-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0616320-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41550067-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"8m8.964683458s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104829638692835328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"open","pr_sha":"d68eeeb0fbe326b55631f88b73b193dece036475","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-29T07:04:26Z","started_at":"2026-09-29T07:04:26Z","completed_at":"2026-09-29T09:23:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:32:17.287782885Z","timing_metadata_checked_at":"2026-09-29T09:32:17.287782885Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:32:17.291178574Z","region_metadata_checked_at":"2026-09-29T09:32:17.291178574Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826606282346496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7221,"pr_state":"closed","pr_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","final_merged_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T06:52:23Z","started_at":"2026-09-29T06:52:23Z","completed_at":"2026-09-29T09:23:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:32:17.748382868Z","timing_metadata_checked_at":"2026-09-29T09:32:17.748382868Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:32:17.751278359Z","region_metadata_checked_at":"2026-09-29T09:32:17.751278359Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ff3e19a31a276ead208fe8106caad32add67dfc4665b64a5eb2a8128c0ae289e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826606282346496","occurred_at":"2026-09-29T06:52:23Z","failed_at":"alert","signature_id":"504b638e5c47bbb7f727b9f7f04de53e5510bb71ec9e87da77e843cd7cf1298d","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T08:03:00Z\n Ended: 2026-09-29T08:11:02Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j2346496-svc\", component=\"backend\", name=\"operationclustercreate\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue operationclustercreate has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826574321750016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"closed","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T06:52:15Z","started_at":"2026-09-29T06:52:15Z","completed_at":"2026-09-29T09:12:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:21:23.669135955Z","timing_metadata_checked_at":"2026-09-29T09:21:23.669135955Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:21:23.672885443Z","region_metadata_checked_at":"2026-09-29T09:21:23.672885443Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T06:52:10Z","started_at":"2026-09-29T06:52:10Z","completed_at":"2026-09-29T09:23:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:32:16.860816392Z","timing_metadata_checked_at":"2026-09-29T09:32:16.860816392Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:32:16.864003481Z","region_metadata_checked_at":"2026-09-29T09:32:16.864003481Z"},"failed_at":["alert","e2e"],"failed_test_count":17,"occurrences":[{"row_id":"1aa507f6968236ac3acfddf595eada4bd114e4b3c0a8ac35d7117a3f499079c6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"1a8ec8671438f853ebb1aa3d567d2ac4a7860a97fa31211460e567cab9ec6674","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-23-nv2hh5\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fea200\u003e: \n failed to create HCP cluster cluster-zstream-4-23-nv2hh5: failed waiting for cluster=\"cluster-zstream-4-23-nv2hh5\" in resourcegroup=\"rg-zstream-upgrade-4-23-s5jlm945vrgj\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24c87b15-b875-4d01-b23b-af0f96b39367\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24c87b15-b875-4d01-b23b-af0f96b39367\",\n \"name\": \"24c87b15-b875-4d01-b23b-af0f96b39367\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:20.715337051Z\",\n \"endTime\": \"2026-09-29T08:24:26.999132889Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-23-nv2hh5\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fea200\u003e: \n failed to create HCP cluster cluster-zstream-4-23-nv2hh5: failed waiting for cluster=\"cluster-zstream-4-23-nv2hh5\" in resourcegroup=\"rg-zstream-upgrade-4-23-s5jlm945vrgj\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24c87b15-b875-4d01-b23b-af0f96b39367\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/24c87b15-b875-4d01-b23b-af0f96b39367\",\n \"name\": \"24c87b15-b875-4d01-b23b-af0f96b39367\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:20.715337051Z\",\n \"endTime\": \"2026-09-29T08:24:26.999132889Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1f7bb70e90d6f3db4b2bd43b2df3a0f77a58a24c44ab66ff48e1f5563f046cb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"8adb242a1fc2de6b404cd3d90c49c42a5b81fe7628eac64c0600f4cf8d52de93","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-blp9q9\" with version 4.22.15 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d622c0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-blp9q9: failed waiting for cluster=\"cluster-zstream-4-22-blp9q9\" in resourcegroup=\"rg-zstream-upgrade-4-22-fmzcbkj4sb6q\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b4848195-94a2-48df-b91a-b769bff1b68c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b4848195-94a2-48df-b91a-b769bff1b68c\",\n \"name\": \"b4848195-94a2-48df-b91a-b769bff1b68c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:50.292245877Z\",\n \"endTime\": \"2026-09-29T08:24:57.452096679Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-blp9q9\" with version 4.22.15 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d622c0\u003e: \n failed to create HCP cluster cluster-zstream-4-22-blp9q9: failed waiting for cluster=\"cluster-zstream-4-22-blp9q9\" in resourcegroup=\"rg-zstream-upgrade-4-22-fmzcbkj4sb6q\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b4848195-94a2-48df-b91a-b769bff1b68c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b4848195-94a2-48df-b91a-b769bff1b68c\",\n \"name\": \"b4848195-94a2-48df-b91a-b769bff1b68c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:50.292245877Z\",\n \"endTime\": \"2026-09-29T08:24:57.452096679Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"237d739f941fe4dfaa02d47c88a98c5e252918c9b469d10d6cdde653693d514e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"alert","signature_id":"f0c78a6caa6975649b256d5a71645e6f52b807cad37e11f49a6fb64123d1ce91","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T08:30:07Z\n Ended: 2026-09-29T08:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i23bv73ftklp9paqra2ef1uvhfhih-z2g1x5t0y3e0n2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i23bv73ftklp9paqra2ef1uvhfhih-z2g1x5t0y3e0n2m/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T08:30:07Z\n Ended: 2026-09-29T08:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1qgjm100jlb0kv1qg3ij1gftkogs-n1u9p0c1i8f8h1v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i1qgjm100jlb0kv1qg3ij1gftkogs-n1u9p0c1i8f8h1v/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T08:30:07Z\n Ended: 2026-09-29T08:48:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1v634q59ghjdn03ih5nveqi92bne-b3q8z4x9q6s0r6o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i1v634q59ghjdn03ih5nveqi92bne-b3q8z4x9q6s0r6o/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-29T08:30:08Z\n Ended: 2026-09-29T08:48:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2598lplvrsf0s9b0lnetgap7c3pm-acr-pull-create\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i2598lplvrsf0s9b0lnetgap7c3pm-acr-pull-create/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-29T08:30:08Z\n Ended: 2026-09-29T08:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i21j3vg3ak4ep8960ujatt7mie9ip-ea-listhcr4j8\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i21j3vg3ak4ep8960ujatt7mie9ip-ea-listhcr4j8/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-29T08:31:08Z\n Ended: 2026-09-29T08:48:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1nohuspbu2tqvi3lqnaqjiqmepii-u0v7m3x2h5g5c3y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i1nohuspbu2tqvi3lqnaqjiqmepii-u0v7m3x2h5g5c3y/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-29T08:31:08Z\n Ended: 2026-09-29T08:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i21lp6qql74kuc0jeund2dp8k3e5o-u1w7f0g9h5z6l6n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i21lp6qql74kuc0jeund2dp8k3e5o-u1w7f0g9h5z6l6n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-29T08:31:08Z\n Ended: 2026-09-29T08:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2fc1poue2ub3f5n014gqkv4othuk-r1v9h5p1o9x0i5z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i2fc1poue2ub3f5n014gqkv4othuk-r1v9h5p1o9x0i5z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-29T08:31:08Z\n Ended: 2026-09-29T08:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2b1m61e1uvlskbah1j29ct4ov9km-g0u9n1p2u6q7n2c\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i2b1m61e1uvlskbah1j29ct4ov9km-g0u9n1p2u6q7n2c/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-29T08:31:08Z\n Ended: 2026-09-29T08:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2fclju63g3kb2c1qlbdm3tgnarp2-z5b2k1s2b2d6r6p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i2fclju63g3kb2c1qlbdm3tgnarp2-z5b2k1s2b2d6r6p/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-29T08:32:10Z\n Ended: 2026-09-29T08:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1q19vd24pc2dma39mndkh6d055c3-u8o7e3g2q8p1m7w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i1q19vd24pc2dma39mndkh6d055c3-u8o7e3g2q8p1m7w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-29T08:33:07Z\n Ended: 2026-09-29T08:50:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2j0k2qc17trgshchrbb31b6r0e12-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t4i2j0k2qc17trgshchrbb31b6r0e12-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"2736fd980d6aa705fb6fe67952e1046fd5c01080e5565dc3495bf4022bb9024f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"b3fc4400ca25570010952257c9a8a56d0ec6da00c5dff3352e2ab0590de1e3d0","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f455c0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-w69hpslshwlv\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a74e2c6d-6e42-40b4-a175-1ed6d31462aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a74e2c6d-6e42-40b4-a175-1ed6d31462aa\",\n \"name\": \"a74e2c6d-6e42-40b4-a175-1ed6d31462aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:43.790638899Z\",\n \"endTime\": \"2026-09-29T08:25:46.7532113Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f455c0\u003e: \n failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-w69hpslshwlv\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a74e2c6d-6e42-40b4-a175-1ed6d31462aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a74e2c6d-6e42-40b4-a175-1ed6d31462aa\",\n \"name\": \"a74e2c6d-6e42-40b4-a175-1ed6d31462aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:43.790638899Z\",\n \"endTime\": \"2026-09-29T08:25:46.7532113Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"279261d3e7c954ac78d5d7e2926a5be7fd1fb5bebdec192b55043ac0500f4e91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"258610727f9484f38009b3adf5c6a4077b893b90a433eb7ef45ccb9c9de1a69e","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0cd80\u003e: \n failed to create HCP cluster np-downgrade-d5kpcc: failed waiting for cluster=\"np-downgrade-d5kpcc\" in resourcegroup=\"rg-np-downgrade-d5kpcc-spqwkkbnns2k\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efc92f38-b31c-40d9-a2e8-d27812fb806d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efc92f38-b31c-40d9-a2e8-d27812fb806d\",\n \"name\": \"efc92f38-b31c-40d9-a2e8-d27812fb806d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T08:18:50.329063599Z\",\n \"endTime\": \"2026-09-29T08:47:53.988905635Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:582]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0cd80\u003e: \n failed to create HCP cluster np-downgrade-d5kpcc: failed waiting for cluster=\"np-downgrade-d5kpcc\" in resourcegroup=\"rg-np-downgrade-d5kpcc-spqwkkbnns2k\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efc92f38-b31c-40d9-a2e8-d27812fb806d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/efc92f38-b31c-40d9-a2e8-d27812fb806d\",\n \"name\": \"efc92f38-b31c-40d9-a2e8-d27812fb806d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T08:18:50.329063599Z\",\n \"endTime\": \"2026-09-29T08:47:53.988905635Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"31c7ba9bab8ab0c4d9de9ef63693088f657d7bb9acf5fb2704d117295f71ee39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"7dad6f696b4df7e67ff8dd1a260da1c21fedb51414758a6ad06a40b14304b4ea","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005d89e0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-dnvbvpskzmvj\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1748606c-047b-472e-ba7c-0674ebb0168b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1748606c-047b-472e-ba7c-0674ebb0168b\",\n \"name\": \"1748606c-047b-472e-ba7c-0674ebb0168b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:56.231939427Z\",\n \"endTime\": \"2026-09-29T08:24:57.501320567Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:83]: failed to create HCP cluster np-update-nodes-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005d89e0\u003e: \n failed to create HCP cluster np-update-nodes-hcp-cluster: failed waiting for cluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-dnvbvpskzmvj\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1748606c-047b-472e-ba7c-0674ebb0168b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1748606c-047b-472e-ba7c-0674ebb0168b\",\n \"name\": \"1748606c-047b-472e-ba7c-0674ebb0168b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:56.231939427Z\",\n \"endTime\": \"2026-09-29T08:24:57.501320567Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"50d21b8cf503b558f7b3f928fa29c567038bc5f46a0c43fe4a6c2f1708481c1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"adef546dfe1912edec9c55e712cff4086be816fb3a8d702ad01ea5e76d6bf367","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001140d40\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-thrgdc65vqpn\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b4f9cc11-1074-4eb5-9072-3aea0ca2edff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b4f9cc11-1074-4eb5-9072-3aea0ca2edff\",\n \"name\": \"b4f9cc11-1074-4eb5-9072-3aea0ca2edff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:04.7919176Z\",\n \"endTime\": \"2026-09-29T08:25:06.974828638Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001140d40\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-thrgdc65vqpn\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b4f9cc11-1074-4eb5-9072-3aea0ca2edff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b4f9cc11-1074-4eb5-9072-3aea0ca2edff\",\n \"name\": \"b4f9cc11-1074-4eb5-9072-3aea0ca2edff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:04.7919176Z\",\n \"endTime\": \"2026-09-29T08:25:06.974828638Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6ba3e9bbac91de5bec677c35d2df167aced59c89a4e6dc3fa101f60f5ddd2aa9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"8dc02c43b57c0ee03aeef88ae4ad57d6ae9fd39d4802670736bdf7a64393aa10","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:96]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef8600\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rztjl5zww876\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38a4b2b5-8146-4c72-8081-b14b58782b99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38a4b2b5-8146-4c72-8081-b14b58782b99\",\n \"name\": \"38a4b2b5-8146-4c72-8081-b14b58782b99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:27.062951858Z\",\n \"endTime\": \"2026-09-29T08:25:27.200284072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:96]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef8600\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rztjl5zww876\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38a4b2b5-8146-4c72-8081-b14b58782b99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/38a4b2b5-8146-4c72-8081-b14b58782b99\",\n \"name\": \"38a4b2b5-8146-4c72-8081-b14b58782b99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:27.062951858Z\",\n \"endTime\": \"2026-09-29T08:25:27.200284072Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6cddf86d04c35ec5179b0529ca5d0a56202d482e458e80dd845d1edec7b1d030","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"0d83436293800226028bcae9c08678862f780cfcb011a3356701b7aed9045191","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-lct6jp\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00172c540\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-lct6jp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-lct6jp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-ngktdpgf5hqn\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a12c82d-272c-4787-9f51-76fc1e3b720d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a12c82d-272c-4787-9f51-76fc1e3b720d\",\n \"name\": \"7a12c82d-272c-4787-9f51-76fc1e3b720d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:45.445577888Z\",\n \"endTime\": \"2026-09-29T08:25:46.887707927Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-21-lct6jp\" at install version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00172c540\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-21-lct6jp: failed waiting for cluster=\"cp-ystream-upgrade-4-21-lct6jp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-ngktdpgf5hqn\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a12c82d-272c-4787-9f51-76fc1e3b720d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7a12c82d-272c-4787-9f51-76fc1e3b720d\",\n \"name\": \"7a12c82d-272c-4787-9f51-76fc1e3b720d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:45.445577888Z\",\n \"endTime\": \"2026-09-29T08:25:46.887707927Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"717248ddd81a788a1ea481fa67251e1015226305059b63be45ea34de3e0f57f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"6b20055d9357902c77da478c0ba5a91a54716b21bf059c3cf84848cd99cd01de","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-rbgmsv\" with version 4.20.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009162c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-rbgmsv: failed waiting for cluster=\"cluster-zstream-4-20-rbgmsv\" in resourcegroup=\"rg-zstream-upgrade-4-20-fcxvqrgtmc96\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3efe8dcf-f0bf-4bc9-9b37-fd2d80e46279\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3efe8dcf-f0bf-4bc9-9b37-fd2d80e46279\",\n \"name\": \"3efe8dcf-f0bf-4bc9-9b37-fd2d80e46279\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:39.993617906Z\",\n \"endTime\": \"2026-09-29T08:24:46.753100751Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-rbgmsv\" with version 4.20.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009162c0\u003e: \n failed to create HCP cluster cluster-zstream-4-20-rbgmsv: failed waiting for cluster=\"cluster-zstream-4-20-rbgmsv\" in resourcegroup=\"rg-zstream-upgrade-4-20-fcxvqrgtmc96\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3efe8dcf-f0bf-4bc9-9b37-fd2d80e46279\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3efe8dcf-f0bf-4bc9-9b37-fd2d80e46279\",\n \"name\": \"3efe8dcf-f0bf-4bc9-9b37-fd2d80e46279\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:39.993617906Z\",\n \"endTime\": \"2026-09-29T08:24:46.753100751Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"76bea12ea4d437ae9c2867fe603a7408e67c8f07ad0ce3ce66896cde1a5c1eb0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"alert","signature_id":"543680892a99c06f5433505067858dcca4c238b213ac601887f2f5092b80ac63","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s) (12 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T08:06:04Z\n Ended: 2026-09-29T08:48:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i23bv73ftklp9paqra2ef1uvhfhih-z2g1x5t0y3e0n2m\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i23bv73ftklp9paqra2ef1uvhfhih-z2g1x5t0y3e0n2m/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T08:06:04Z\n Ended: 2026-09-29T08:48:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i1qgjm100jlb0kv1qg3ij1gftkogs-n1u9p0c1i8f8h1v\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i1qgjm100jlb0kv1qg3ij1gftkogs-n1u9p0c1i8f8h1v/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T08:06:04Z\n Ended: 2026-09-29T08:43:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i1v634q59ghjdn03ih5nveqi92bne-b3q8z4x9q6s0r6o\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i1v634q59ghjdn03ih5nveqi92bne-b3q8z4x9q6s0r6o/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-29T08:06:04Z\n Ended: 2026-09-29T08:48:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i2fc1poue2ub3f5n014gqkv4othuk-r1v9h5p1o9x0i5z\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i2fc1poue2ub3f5n014gqkv4othuk-r1v9h5p1o9x0i5z/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-29T08:06:04Z\n Ended: 2026-09-29T08:48:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i21j3vg3ak4ep8960ujatt7mie9ip-ea-listhcr4j8\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i21j3vg3ak4ep8960ujatt7mie9ip-ea-listhcr4j8/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-29T08:06:04Z\n Ended: 2026-09-29T08:48:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i2b1m61e1uvlskbah1j29ct4ov9km-g0u9n1p2u6q7n2c\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i2b1m61e1uvlskbah1j29ct4ov9km-g0u9n1p2u6q7n2c/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-29T08:06:04Z\n Ended: 2026-09-29T08:43:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i2598lplvrsf0s9b0lnetgap7c3pm-acr-pull-create\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i2598lplvrsf0s9b0lnetgap7c3pm-acr-pull-create/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-29T08:07:05Z\n Ended: 2026-09-29T08:43:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i1nohuspbu2tqvi3lqnaqjiqmepii-u0v7m3x2h5g5c3y\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i1nohuspbu2tqvi3lqnaqjiqmepii-u0v7m3x2h5g5c3y/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-29T08:07:05Z\n Ended: 2026-09-29T08:48:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i21lp6qql74kuc0jeund2dp8k3e5o-u1w7f0g9h5z6l6n\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i21lp6qql74kuc0jeund2dp8k3e5o-u1w7f0g9h5z6l6n/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-29T08:07:06Z\n Ended: 2026-09-29T08:48:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i2fclju63g3kb2c1qlbdm3tgnarp2-z5b2k1s2b2d6r6p\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i2fclju63g3kb2c1qlbdm3tgnarp2-z5b2k1s2b2d6r6p/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-29T08:08:06Z\n Ended: 2026-09-29T08:48:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i1q19vd24pc2dma39mndkh6d055c3-u8o7e3g2q8p1m7w\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i1q19vd24pc2dma39mndkh6d055c3-u8o7e3g2q8p1m7w/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-29T08:08:06Z\n Ended: 2026-09-29T08:48:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4i2j0k2qc17trgshchrbb31b6r0e12-cilium-cluster\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4i2j0k2qc17trgshchrbb31b6r0e12-cilium-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"8614c6a4571e6b443c0268a5d883d59b49439214697358299f45188c281cc3c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"alert","signature_id":"c588d03a188306b4e401a20a58a44ba9852bf37fac64c5fb720d3c5bfd21b821","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 24 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T08:16:09Z\n Ended: 2026-09-29T08:44:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i23bv73ftklp9paqra2ef1uvhfhih-z2g1x5t0y3e0n2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i23bv73ftklp9paqra2ef1uvhfhih-z2g1x5t0y3e0n2m/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T08:16:09Z\n Ended: 2026-09-29T08:43:04Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1v634q59ghjdn03ih5nveqi92bne-b3q8z4x9q6s0r6o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i1v634q59ghjdn03ih5nveqi92bne-b3q8z4x9q6s0r6o/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T08:16:09Z\n Ended: 2026-09-29T08:48:05Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i23bv73ftklp9paqra2ef1uvhfhih-z2g1x5t0y3e0n2m\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i23bv73ftklp9paqra2ef1uvhfhih-z2g1x5t0y3e0n2m/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-29T08:16:09Z\n Ended: 2026-09-29T08:43:04Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1v634q59ghjdn03ih5nveqi92bne-b3q8z4x9q6s0r6o\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i1v634q59ghjdn03ih5nveqi92bne-b3q8z4x9q6s0r6o/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-29T08:16:10Z\n Ended: 2026-09-29T08:44:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1qgjm100jlb0kv1qg3ij1gftkogs-n1u9p0c1i8f8h1v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i1qgjm100jlb0kv1qg3ij1gftkogs-n1u9p0c1i8f8h1v/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-29T08:16:10Z\n Ended: 2026-09-29T08:44:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i21j3vg3ak4ep8960ujatt7mie9ip-ea-listhcr4j8\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i21j3vg3ak4ep8960ujatt7mie9ip-ea-listhcr4j8/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-29T08:16:10Z\n Ended: 2026-09-29T08:48:05Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1qgjm100jlb0kv1qg3ij1gftkogs-n1u9p0c1i8f8h1v\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i1qgjm100jlb0kv1qg3ij1gftkogs-n1u9p0c1i8f8h1v/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-29T08:16:10Z\n Ended: 2026-09-29T08:44:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i21j3vg3ak4ep8960ujatt7mie9ip-ea-listhcr4j8\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i21j3vg3ak4ep8960ujatt7mie9ip-ea-listhcr4j8/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-29T08:16:10Z\n Ended: 2026-09-29T08:48:06Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2b1m61e1uvlskbah1j29ct4ov9km-g0u9n1p2u6q7n2c\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2b1m61e1uvlskbah1j29ct4ov9km-g0u9n1p2u6q7n2c/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-29T08:16:10Z\n Ended: 2026-09-29T08:43:04Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2598lplvrsf0s9b0lnetgap7c3pm-acr-pull-create\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2598lplvrsf0s9b0lnetgap7c3pm-acr-pull-create/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-29T08:16:12Z\n Ended: 2026-09-29T08:48:06Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2fc1poue2ub3f5n014gqkv4othuk-r1v9h5p1o9x0i5z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2fc1poue2ub3f5n014gqkv4othuk-r1v9h5p1o9x0i5z/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-29T08:16:12Z\n Ended: 2026-09-29T08:44:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2fc1poue2ub3f5n014gqkv4othuk-r1v9h5p1o9x0i5z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2fc1poue2ub3f5n014gqkv4othuk-r1v9h5p1o9x0i5z/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-29T08:16:12Z\n Ended: 2026-09-29T08:43:04Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2598lplvrsf0s9b0lnetgap7c3pm-acr-pull-create\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2598lplvrsf0s9b0lnetgap7c3pm-acr-pull-create/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-29T08:17:07Z\n Ended: 2026-09-29T08:43:04Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1nohuspbu2tqvi3lqnaqjiqmepii-u0v7m3x2h5g5c3y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i1nohuspbu2tqvi3lqnaqjiqmepii-u0v7m3x2h5g5c3y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-29T08:17:07Z\n Ended: 2026-09-29T08:43:04Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1nohuspbu2tqvi3lqnaqjiqmepii-u0v7m3x2h5g5c3y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i1nohuspbu2tqvi3lqnaqjiqmepii-u0v7m3x2h5g5c3y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-29T08:17:07Z\n Ended: 2026-09-29T08:48:05Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i21lp6qql74kuc0jeund2dp8k3e5o-u1w7f0g9h5z6l6n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i21lp6qql74kuc0jeund2dp8k3e5o-u1w7f0g9h5z6l6n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-29T08:17:07Z\n Ended: 2026-09-29T08:44:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2fclju63g3kb2c1qlbdm3tgnarp2-z5b2k1s2b2d6r6p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2fclju63g3kb2c1qlbdm3tgnarp2-z5b2k1s2b2d6r6p/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-29T08:17:08Z\n Ended: 2026-09-29T08:48:05Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2fclju63g3kb2c1qlbdm3tgnarp2-z5b2k1s2b2d6r6p\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2fclju63g3kb2c1qlbdm3tgnarp2-z5b2k1s2b2d6r6p/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-29T08:17:08Z\n Ended: 2026-09-29T08:48:06Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2b1m61e1uvlskbah1j29ct4ov9km-g0u9n1p2u6q7n2c\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2b1m61e1uvlskbah1j29ct4ov9km-g0u9n1p2u6q7n2c/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-29T08:17:08Z\n Ended: 2026-09-29T08:44:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i21lp6qql74kuc0jeund2dp8k3e5o-u1w7f0g9h5z6l6n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i21lp6qql74kuc0jeund2dp8k3e5o-u1w7f0g9h5z6l6n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-29T08:18:08Z\n Ended: 2026-09-29T08:48:05Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2j0k2qc17trgshchrbb31b6r0e12-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2j0k2qc17trgshchrbb31b6r0e12-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-29T08:18:09Z\n Ended: 2026-09-29T08:48:06Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1q19vd24pc2dma39mndkh6d055c3-u8o7e3g2q8p1m7w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i1q19vd24pc2dma39mndkh6d055c3-u8o7e3g2q8p1m7w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-29T08:18:09Z\n Ended: 2026-09-29T08:44:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i2j0k2qc17trgshchrbb31b6r0e12-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i2j0k2qc17trgshchrbb31b6r0e12-cilium-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-29T08:18:09Z\n Ended: 2026-09-29T08:44:10Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2126208-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.203:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2126208/providers/microsoft.monitor/accounts/hcps-j2126208\", namespace=\"ocm-arohcpci01-2t4i1q19vd24pc2dma39mndkh6d055c3-u8o7e3g2q8p1m7w\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-njht2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4i1q19vd24pc2dma39mndkh6d055c3-u8o7e3g2q8p1m7w/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"a3fbd5dbbd24ebf6417a77e29fd284217d361df03a8dd5fe5b90b553f74950f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"d3dadc9039a42b1c903733e63e869b37b336323bac8aa107c0c481d14404d703","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-ndbj46 at version 4.22.16\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001884660\u003e: \n failed to create HCP cluster np-skip-minor-ndbj46: failed waiting for cluster=\"np-skip-minor-ndbj46\" in resourcegroup=\"rg-np-skip-minor-ndbj46-m78qhj5gv2zn\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22160fab-8cb1-439f-bd94-2c14e042c50e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22160fab-8cb1-439f-bd94-2c14e042c50e\",\n \"name\": \"22160fab-8cb1-439f-bd94-2c14e042c50e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T08:17:10.081306885Z\",\n \"endTime\": \"2026-09-29T08:46:13.98134671Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:443]: failed to create HCP cluster np-skip-minor-ndbj46 at version 4.22.16\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001884660\u003e: \n failed to create HCP cluster np-skip-minor-ndbj46: failed waiting for cluster=\"np-skip-minor-ndbj46\" in resourcegroup=\"rg-np-skip-minor-ndbj46-m78qhj5gv2zn\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22160fab-8cb1-439f-bd94-2c14e042c50e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/22160fab-8cb1-439f-bd94-2c14e042c50e\",\n \"name\": \"22160fab-8cb1-439f-bd94-2c14e042c50e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T08:17:10.081306885Z\",\n \"endTime\": \"2026-09-29T08:46:13.98134671Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a4b89c7b63008a70f8d1a027008f5018c2772a39c3222d2164df5b9df3d00495","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"2ebf23239aa1ebb46a5e91a1a846755a6f3fd4df0141fe3a351c32292eab5f38","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-5jrw9b8n5g28/cluster-candidate-4-23-nj55c2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f20440\u003e: \n failed to create HCP cluster cluster-candidate-4-23-nj55c2: failed waiting for cluster=\"cluster-candidate-4-23-nj55c2\" in resourcegroup=\"rg-candidate-4-23-5jrw9b8n5g28\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd2432b1-d41f-4d67-9e36-d7401cd5dd0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd2432b1-d41f-4d67-9e36-d7401cd5dd0a\",\n \"name\": \"cd2432b1-d41f-4d67-9e36-d7401cd5dd0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:10.422882179Z\",\n \"endTime\": \"2026-09-29T08:24:17.034241527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-5jrw9b8n5g28/cluster-candidate-4-23-nj55c2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f20440\u003e: \n failed to create HCP cluster cluster-candidate-4-23-nj55c2: failed waiting for cluster=\"cluster-candidate-4-23-nj55c2\" in resourcegroup=\"rg-candidate-4-23-5jrw9b8n5g28\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd2432b1-d41f-4d67-9e36-d7401cd5dd0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd2432b1-d41f-4d67-9e36-d7401cd5dd0a\",\n \"name\": \"cd2432b1-d41f-4d67-9e36-d7401cd5dd0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:10.422882179Z\",\n \"endTime\": \"2026-09-29T08:24:17.034241527Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cd2240d0ef0f19f9b3fee0adb7dba96ef8779b383f0e13996b0a07991255d2e7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"467f22b54a8e729892ab9ebdc405f18f8be06c724be9b496711b9eac7a5ab5e4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-k7lpbdwsc5kj/cluster-candidate-4-20-s8nwkw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b56160\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s8nwkw: failed waiting for cluster=\"cluster-candidate-4-20-s8nwkw\" in resourcegroup=\"rg-candidate-4-20-k7lpbdwsc5kj\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c6eb1e-c4d9-4201-afcd-e3f4592a25ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c6eb1e-c4d9-4201-afcd-e3f4592a25ea\",\n \"name\": \"b0c6eb1e-c4d9-4201-afcd-e3f4592a25ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:19.424941126Z\",\n \"endTime\": \"2026-09-29T08:24:26.872256015Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-20-k7lpbdwsc5kj/cluster-candidate-4-20-s8nwkw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b56160\u003e: \n failed to create HCP cluster cluster-candidate-4-20-s8nwkw: failed waiting for cluster=\"cluster-candidate-4-20-s8nwkw\" in resourcegroup=\"rg-candidate-4-20-k7lpbdwsc5kj\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c6eb1e-c4d9-4201-afcd-e3f4592a25ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b0c6eb1e-c4d9-4201-afcd-e3f4592a25ea\",\n \"name\": \"b0c6eb1e-c4d9-4201-afcd-e3f4592a25ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:19.424941126Z\",\n \"endTime\": \"2026-09-29T08:24:26.872256015Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f5b9cebc7255ff62c2f279a11ea08884188284c3272a583b1293fe30a67b55ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"0be071bd8e0c47562898ef2db89ba6ce80590828d2458bd98774983dd66b16c0","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ace0\u003e: \n failed to create HCP cluster ea-listhcr4j8: failed waiting for cluster=\"ea-listhcr4j8\" in resourcegroup=\"ea-list-dlsgl8fbqcls\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/209ad390-8ddf-4c8a-b8cb-985b84178431\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/209ad390-8ddf-4c8a-b8cb-985b84178431\",\n \"name\": \"209ad390-8ddf-4c8a-b8cb-985b84178431\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:50.088481005Z\",\n \"endTime\": \"2026-09-29T08:24:57.633654763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:87]: failed to create HCP cluster for external auth list test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c6ace0\u003e: \n failed to create HCP cluster ea-listhcr4j8: failed waiting for cluster=\"ea-listhcr4j8\" in resourcegroup=\"ea-list-dlsgl8fbqcls\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/209ad390-8ddf-4c8a-b8cb-985b84178431\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/209ad390-8ddf-4c8a-b8cb-985b84178431\",\n \"name\": \"209ad390-8ddf-4c8a-b8cb-985b84178431\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:55:50.088481005Z\",\n \"endTime\": \"2026-09-29T08:24:57.633654763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"fdfcf0b881b64abc0a3873d00caaf5f551d8a62c4f20ab0a60ee98d41e072395","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826553132126208","occurred_at":"2026-09-29T06:52:10Z","failed_at":"e2e","signature_id":"c55c1b64c511f98c08b0977dfcd60dfddbdcd845baf654e7b73f3f31dbfaa837","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b42c60\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-cjnzsbmj87c2\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c956b498-2960-48f7-9e3a-45091246e492\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c956b498-2960-48f7-9e3a-45091246e492\",\n \"name\": \"c956b498-2960-48f7-9e3a-45091246e492\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:59.880131402Z\",\n \"endTime\": \"2026-09-29T08:26:06.763498601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b42c60\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-cjnzsbmj87c2\" to finish creating: GET https://rp.j2126208.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c956b498-2960-48f7-9e3a-45091246e492\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c956b498-2960-48f7-9e3a-45091246e492\",\n \"name\": \"c956b498-2960-48f7-9e3a-45091246e492\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T07:56:59.880131402Z\",\n \"endTime\": \"2026-09-29T08:26:06.763498601Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":17,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54"],"matched_occurrences":17,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826523797164032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T06:52:03Z","started_at":"2026-09-29T06:52:03Z","completed_at":"2026-09-29T07:46:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T07:56:51.360684702Z","timing_metadata_checked_at":"2026-09-29T07:56:51.360684702Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T07:56:51.363218094Z","region_metadata_checked_at":"2026-09-29T07:56:51.363218094Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"c948763ce9474faedc321a9830eb3a71108e428069db18669e26f4961999a2e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826523797164032","occurred_at":"2026-09-29T06:52:03Z","failed_at":"provision","signature_id":"559e68ba7a5d0082b9e1b07e6236c31c07348bf49d51ab62138e05cec5713570","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-29T07:31:00.292Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-29T07:39:37.421Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-29T07:31:00.303380614Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-29T07:31:02.254116159Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-29T07:31:02.46653855Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:39:37.418185199Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"C8FA27F7295427D4\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7164032-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j7164032-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:35:51.0771325Z\\\",\\\"duration\\\":\\\"PT1M22.2335838S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:39:37.418275122Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"33A35D458B279097\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7164032-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j7164032-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:35:51.0771325Z\\\",\\\"duration\\\":\\\"PT1M22.6887596S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:39:37.418293622Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"79ADB6C410C9C2E8\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7164032-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j7164032-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:35:51.0771325Z\\\",\\\"duration\\\":\\\"PT2M7.5697899S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:39:37.418312323Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"EDFCA9BCA6672AF2\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7164032-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j7164032-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:35:51.0771325Z\\\",\\\"duration\\\":\\\"PT3M43.9835813S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:39:37.418326893Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1970DDD03D9FA410\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7164032-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j7164032-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-29T07:31:06.0370871Z\\\",\\\"duration\\\":\\\"PT5M19.4774005S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-76862120-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-76862120-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-29T07:39:37.418416366Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"48E4059464328ACA\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7164032-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:31:06.0370871Z\\\",\\\"duration\\\":\\\"PT4M44.9619682S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:39:37.418445817Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7164032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7164032-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7164032-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109398228295884\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7164032-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-76862120-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7164032-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-76862120-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"8m37.115069943s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826504838909952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7224,"pr_state":"closed","pr_sha":"a6cabcda34fdee6066dd2b496192fcb45072451a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T06:51:59Z","started_at":"2026-09-29T06:51:59Z","completed_at":"2026-09-29T09:09:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:21:24.108076554Z","timing_metadata_checked_at":"2026-09-29T09:21:24.108076554Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:21:24.110753546Z","region_metadata_checked_at":"2026-09-29T09:21:24.110753546Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"511f3bab8cddfc4e4d30f07da9454bcf899542896e883272e43f84e011422ae7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826504838909952","occurred_at":"2026-09-29T06:51:59Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T07:58:21Z\n Ended: 2026-09-29T08:14:22Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j8909952-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826494294429696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"e7500a3658c1692b6c32705cb7edf403b704aae4","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T06:51:56Z","started_at":"2026-09-29T06:51:56Z","completed_at":"2026-09-29T07:36:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T07:46:13.891570609Z","timing_metadata_checked_at":"2026-09-29T07:46:13.891570609Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T07:46:13.895086599Z","region_metadata_checked_at":"2026-09-29T07:46:13.895086599Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"5f99192cdf1c2ea91d6d8e629981f96710b5119a3805a100758596ee1201da6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104826494294429696","occurred_at":"2026-09-29T06:51:56Z","failed_at":"provision","signature_id":"0c20aea12902dbe5dbe700a97aa1f12ffadd22d1e925f86bb519d2d1889b1205","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-29T07:23:38.273Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-29T07:31:15.555Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-29T07:23:38.281949486Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-29T07:23:40.38543482Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-29T07:23:40.528142719Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:31:15.55271476Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E43B8443956D5BE9\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4429696-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j4429696-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:29:17.7130663Z\\\",\\\"duration\\\":\\\"PT1M54.3992313S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:31:15.552773102Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"99EFDFB8BE97BA0C\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4429696-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j4429696-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:29:17.7130663Z\\\",\\\"duration\\\":\\\"PT1M21.9267324S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:31:15.552787782Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"6E8F5566033BE306\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4429696-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j4429696-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:23:43.8173291Z\\\",\\\"duration\\\":\\\"PT7M10.6693253S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:31:15.552798932Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"BF946B39B0195954\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4429696-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j4429696-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-29T07:29:17.7130663Z\\\",\\\"duration\\\":\\\"PT35.1152298S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-35477505-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-35477505-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-29T07:31:15.552851723Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"7C0D49E72643FEE7\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4429696-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j4429696-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:29:17.7130663Z\\\",\\\"duration\\\":\\\"PT1M53.1290152S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:31:15.552864173Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"9C641B1653694FC3\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4429696-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:23:43.8173291Z\\\",\\\"duration\\\":\\\"PT5M33.8032912S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:31:15.552894944Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4429696-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4429696-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4429696-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109402647707877\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4429696-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-35477505-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4429696-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-35477505-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m37.270946158s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104819949015928832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7073,"pr_state":"open","pr_sha":"b9d120f5fe68fda8bac75fac89516e32b56c6e6e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T06:25:56Z","started_at":"2026-09-29T06:25:56Z","completed_at":"2026-09-29T07:18:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T07:25:20.800332691Z","timing_metadata_checked_at":"2026-09-29T07:25:20.800332691Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T07:25:20.803503881Z","region_metadata_checked_at":"2026-09-29T07:25:20.803503881Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e161e2e8f08e47ea66f545efc0da44fa98d317f08378abbe6f12da20a0f42929","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104819949015928832","occurred_at":"2026-09-29T06:25:56Z","failed_at":"provision","signature_id":"a0c2f4d38e32bdfedb562acec87bed9ccb4b418df192cf3ec74daa4581418e62","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-29T07:03:34.990Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-29T07:10:44.777Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-29T07:03:34.999232343Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-29T07:03:37.116357125Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-29T07:03:37.313150533Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:10:44.775142959Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1B5A18D5707F4650\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5928832-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j5928832-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:03:42.5803863Z\\\",\\\"duration\\\":\\\"PT6M56.205703S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:10:44.77520321Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"9A546005D0D62A98\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5928832-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j5928832-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:08:45.1132117Z\\\",\\\"duration\\\":\\\"PT1M39.0493131S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:10:44.775216231Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2499AE04FF928A50\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5928832-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j5928832-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-29T07:08:45.1132117Z\\\",\\\"duration\\\":\\\"PT35.3167623S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25953470-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25953470-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-29T07:10:44.775267692Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2CFA13A566272FDC\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5928832-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j5928832-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:08:45.1132117Z\\\",\\\"duration\\\":\\\"PT1M53.5752914S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:10:44.775292343Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"61A3D0CD83893682\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5928832-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j5928832-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:08:45.1132117Z\\\",\\\"duration\\\":\\\"PT1M21.924003S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:10:44.775306823Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"ED79FE7C44841137\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5928832-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-29T07:03:42.5803863Z\\\",\\\"duration\\\":\\\"PT5M2.4596624S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-29T07:10:44.775326944Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5928832-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5928832-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5928832-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109414679431690\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5928832-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25953470-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5928832-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25953470-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m9.77610027s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104818295738732544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7225,"pr_state":"closed","pr_sha":"72e9f5a561068e3526344d3fb03a5ceee6030608","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-29T06:19:21Z","started_at":"2026-09-29T06:19:21Z","completed_at":"2026-09-29T08:42:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T08:50:18.888231129Z","timing_metadata_checked_at":"2026-09-29T08:50:18.888231129Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T08:50:18.891228419Z","region_metadata_checked_at":"2026-09-29T08:50:18.891228419Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104807316128796672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6746,"pr_state":"open","pr_sha":"701c84e64f23891954e1dc625eeb61c094c3cbba","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T05:35:44Z","started_at":"2026-09-29T05:35:44Z","completed_at":"2026-09-29T07:54:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T07:56:51.208037585Z","timing_metadata_checked_at":"2026-09-29T07:56:51.208037585Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T07:56:51.212420571Z","region_metadata_checked_at":"2026-09-29T07:56:51.212420571Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"74efedaa5be11aef67fb620f8478bd5a454074aa66e6c483209c08fe3b396c20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104807316128796672","occurred_at":"2026-09-29T05:35:44Z","failed_at":"e2e","signature_id":"0bee2919a9bee3776a8fd283c56ec640f998df0f4d9e0ac0bb7806de6e4b2a76","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:170]: private ingress should not be reachable from outside the VNet, but connection succeeded\nExpected an error to have occurred. Got:\n \u003cnil\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365","failure_pattern":"private ingress should not be reachable from outside the VNet, but connection succeeded","search_query":"private ingress should not be reachable from outside the VNet, but connection succeeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["406d47e638f407d24104655febf18e753de33546f725682f49558d9bc4ae2365"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"dae44384d4cdd5fe9ce7131eb12cbc6a5612418a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T01:55:42Z","started_at":"2026-09-29T01:55:42Z","completed_at":"2026-09-29T04:39:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:48:01.419481383Z","timing_metadata_checked_at":"2026-09-29T04:48:01.419481383Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:48:01.422260974Z","region_metadata_checked_at":"2026-09-29T04:48:01.422260974Z"},"failed_at":["alert","e2e"],"failed_test_count":10,"occurrences":[{"row_id":"1c0bd5d006c8019c5ff89319af46172bbd67a3c507af3be7252d39ee79319c1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"e2e","signature_id":"a2ea63ce92f1704d7fdc6f20623ccc10334ae26b9406875dd9b289a83e10c1b5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002d68108\u003e: \n resource group \"rg-candidate-4-20-7rmbsrs7gnrh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-rr5qrl\" in resourcegroup=\"rg-candidate-4-20-7rmbsrs7gnrh\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe34cfee-10b3-40d3-9555-85af89de2176\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe34cfee-10b3-40d3-9555-85af89de2176\",\n \"name\": \"fe34cfee-10b3-40d3-9555-85af89de2176\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:31:45.875226211Z\",\n \"endTime\": \"2026-09-29T03:55:49.946038728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dng20u18ftcg2h5cbkv9l2g9r3ti1 still exists (deletion dispatched at 2026-09-29T03:31:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002d68108\u003e: \n resource group \"rg-candidate-4-20-7rmbsrs7gnrh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-rr5qrl\" in resourcegroup=\"rg-candidate-4-20-7rmbsrs7gnrh\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe34cfee-10b3-40d3-9555-85af89de2176\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fe34cfee-10b3-40d3-9555-85af89de2176\",\n \"name\": \"fe34cfee-10b3-40d3-9555-85af89de2176\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:31:45.875226211Z\",\n \"endTime\": \"2026-09-29T03:55:49.946038728Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dng20u18ftcg2h5cbkv9l2g9r3ti1 still exists (deletion dispatched at 2026-09-29T03:31:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1edf8f2d722627fe8a34d080a4bdbae9201378f6d8412665a956421fe56cafc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"alert","signature_id":"f64e8f9ad85cecf345e7661a18548c2672081aaed696567ff8dbbe006b50a916","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T03:34:52Z\n Ended: 2026-09-29T03:55:53Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6245760-svc\", component=\"service-memory\", container=\"aro-hcp-frontend\", namespace=\"aro-hcp\", pod=\"aro-hcp-frontend-588d586db6-sjv52\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-frontend in pod aro-hcp-frontend-588d586db6-sjv52 (namespace aro-hcp) on cluster ci01-j6245760-svc has averaged 159.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-29T03:34:52Z\n Ended: 2026-09-29T03:55:53Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6245760-svc\", component=\"service-memory\", container=\"aro-hcp-frontend\", namespace=\"aro-hcp\", pod=\"aro-hcp-frontend-588d586db6-ksmpx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-frontend in pod aro-hcp-frontend-588d586db6-ksmpx (namespace aro-hcp) on cluster ci01-j6245760-svc has averaged 151.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"2b8244095ba9209bc875a8deaec0392d70d246819a3186d17be7ed03f6f8b1a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"alert","signature_id":"1dcc6a5b20c03697dd44d8cced3748b1a1e6ee8d9a8228477e2148cb6f3c6ce1","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 43 time(s) (16 unknown, 27 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:22:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dng71bgkutvv3n1ceva77lgan1qbb-k0w9z6c2c8q1s2b\", pod=\"kube-apiserver-58ff654cc-2vnlw\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dng71bgkutvv3n1ceva77lgan1qbb-k0w9z6c2c8q1s2b/kube-apiserver-58ff654cc-2vnlw has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:22:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dnda7s25fsc9d65p7r8u1mmci6bd0-g7q0l4a8s0s6g9a\", pod=\"kube-apiserver-85f884b79f-59hnd\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dnda7s25fsc9d65p7r8u1mmci6bd0-g7q0l4a8s0s6g9a/kube-apiserver-85f884b79f-59hnd has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:22:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dng3121cd7fubva8qgdpj2d4qca1t-h4o5g1t0s4b4y4k\", pod=\"dns-operator-55bdff9fb8-rhxht\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dng3121cd7fubva8qgdpj2d4qca1t-h4o5g1t0s4b4y4k/dns-operator-55bdff9fb8-rhxht has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:22:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dng3121cd7fubva8qgdpj2d4qca1t-h4o5g1t0s4b4y4k\", pod=\"ingress-operator-5cb4db558c-pvzkf\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dng3121cd7fubva8qgdpj2d4qca1t-h4o5g1t0s4b4y4k/ingress-operator-5cb4db558c-pvzkf has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:39:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dndg2sdjc743p95i807kb2mn6smbp-g7i6u7a0i4b9m9x\", pod=\"router-5b5f4f9f48-qnm6v\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dndg2sdjc743p95i807kb2mn6smbp-g7i6u7a0i4b9m9x/router-5b5f4f9f48-qnm6v has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:38:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dndnk5camskcuj6c4tgr7nn0f52v3-private-kas\", pod=\"router-b4c79987c-wtm24\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dndnk5camskcuj6c4tgr7nn0f52v3-private-kas/router-b4c79987c-wtm24 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:22:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dng3121cd7fubva8qgdpj2d4qca1t-h4o5g1t0s4b4y4k\", pod=\"cluster-network-operator-5fb7f884f5-vxwlw\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dng3121cd7fubva8qgdpj2d4qca1t-h4o5g1t0s4b4y4k/cluster-network-operator-5fb7f884f5-vxwlw has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:22:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dncr2ht6ka7nkl11md62jh5tldu7e-h1k8q3w9m3q6q9o\", pod=\"cluster-network-operator-7994c466cb-gh5qn\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dncr2ht6ka7nkl11md62jh5tldu7e-h1k8q3w9m3q6q9o/cluster-network-operator-7994c466cb-gh5qn has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:39:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dndg2sdjc743p95i807kb2mn6smbp-g7i6u7a0i4b9m9x\", pod=\"router-5b5f4f9f48-mgvgl\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dndg2sdjc743p95i807kb2mn6smbp-g7i6u7a0i4b9m9x/router-5b5f4f9f48-mgvgl has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:22:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dng3121cd7fubva8qgdpj2d4qca1t-h4o5g1t0s4b4y4k\", pod=\"cluster-image-registry-operator-585d59cb45-hmkgj\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dng3121cd7fubva8qgdpj2d4qca1t-h4o5g1t0s4b4y4k/cluster-image-registry-operator-585d59cb45-hmkgj has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-29T03:16:06Z\n Ended: 2026-09-29T03:22:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dng20u18ftcg2h5cbkv9l2g9r3ti1-x3g1f8t3h0e5e0z\", pod=\"openshift-apiserver-57d55f684-5662d\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dng20u18ftcg2h5cbkv9l2g9r3ti1-x3g1f8t3h0e5e0z/openshift-apiserver-57d55f684-5662d has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-29T03:54:06Z\n Ended: 2026-09-29T04:06:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-kpkhj\", region=\"centralus\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-kpkhj has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-29T03:54:06Z\n Ended: 2026-09-29T04:06:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"set-kubelet-parameters-for-scale-twjmx\", region=\"centralus\", severity=\"warning\"\n Description: Pod kube-system/set-kubelet-parameters-for-scale-twjmx has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-29T03:54:06Z\n Ended: 2026-09-29T04:06:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-52td5\", region=\"centralus\", severity=\"warning\"\n Description: Pod kube-system/aks-secrets-store-csi-driver-52td5 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-09-29T04:19:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-td74p\", region=\"centralus\", severity=\"warning\"\n Description: Pod kube-system/ama-metrics-node-td74p has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-09-29T04:19:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6245760-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"set-kubelet-parameters-for-scale-4bxq9\", region=\"centralus\", severity=\"warning\"\n Description: Pod kube-system/set-kubelet-parameters-for-scale-4bxq9 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"35659b153b21a492f715c4f269c44480b91e23c34c1ea533eb2e4bfd74d71a5c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"e2e","signature_id":"58646d4cd1e2b62950807aed1829ea62c688b06eb7b61d944779e58dacaafba2","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010283c0\u003e: \n resource group \"rg-candidate-4-21-f9jj95dc5hgt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-tb8nxr\" in resourcegroup=\"rg-candidate-4-21-f9jj95dc5hgt\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199fe88f-6a68-4ab4-9cab-d7f75d06d135\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199fe88f-6a68-4ab4-9cab-d7f75d06d135\",\n \"name\": \"199fe88f-6a68-4ab4-9cab-d7f75d06d135\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:30:13.16579683Z\",\n \"endTime\": \"2026-09-29T03:54:19.92105251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dnam4q27mart2dnos2trg042u5fn9 still exists (deletion dispatched at 2026-09-29T03:30:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010283c0\u003e: \n resource group \"rg-candidate-4-21-f9jj95dc5hgt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-tb8nxr\" in resourcegroup=\"rg-candidate-4-21-f9jj95dc5hgt\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199fe88f-6a68-4ab4-9cab-d7f75d06d135\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/199fe88f-6a68-4ab4-9cab-d7f75d06d135\",\n \"name\": \"199fe88f-6a68-4ab4-9cab-d7f75d06d135\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:30:13.16579683Z\",\n \"endTime\": \"2026-09-29T03:54:19.92105251Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dnam4q27mart2dnos2trg042u5fn9 still exists (deletion dispatched at 2026-09-29T03:30:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4090ad3ab7eba65668550e92543b92cb2dee654c82c92169b739fffe7cbea96f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"e2e","signature_id":"1b8d3ab614e86f6e4299eba4ec643d9ad0dae968409a59fbff2be869ff92710d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00120e258\u003e: \n resource group \"rg-candidate-4-22-r5b2hd7c5qpf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-b9vgwv\" in resourcegroup=\"rg-candidate-4-22-r5b2hd7c5qpf\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3afd7a6f-d183-45a9-a85e-c6ad2079ee9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3afd7a6f-d183-45a9-a85e-c6ad2079ee9e\",\n \"name\": \"3afd7a6f-d183-45a9-a85e-c6ad2079ee9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:32:44.381003197Z\",\n \"endTime\": \"2026-09-29T03:56:44.657910184Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dng71bgkutvv3n1ceva77lgan1qbb still exists (deletion dispatched at 2026-09-29T03:32:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00120e258\u003e: \n resource group \"rg-candidate-4-22-r5b2hd7c5qpf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-b9vgwv\" in resourcegroup=\"rg-candidate-4-22-r5b2hd7c5qpf\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3afd7a6f-d183-45a9-a85e-c6ad2079ee9e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3afd7a6f-d183-45a9-a85e-c6ad2079ee9e\",\n \"name\": \"3afd7a6f-d183-45a9-a85e-c6ad2079ee9e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:32:44.381003197Z\",\n \"endTime\": \"2026-09-29T03:56:44.657910184Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dng71bgkutvv3n1ceva77lgan1qbb still exists (deletion dispatched at 2026-09-29T03:32:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"86dc709f42eb0d20204e910cd35c63cf7eac87d1a6c391491b56667e20b0328f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"e2e","signature_id":"343074cfacd08c60e1ef0389bc19cde61f5b2b09eac7a4e3e55f602d1682f23a","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000c3c2b8\u003e: \n resource group \"feature-aggregation-tmknvvnlvscz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-tmknvvnlvscz\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/04b4ca06-437a-4e31-8d71-869db8212330\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/04b4ca06-437a-4e31-8d71-869db8212330\",\n \"name\": \"04b4ca06-437a-4e31-8d71-869db8212330\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:35:24.727596551Z\",\n \"endTime\": \"2026-09-29T03:59:24.666199081Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4docerqoft14tulvhsqagod093vb4h still exists (deletion dispatched at 2026-09-29T03:35:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000c3c2b8\u003e: \n resource group \"feature-aggregation-tmknvvnlvscz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-tmknvvnlvscz\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/04b4ca06-437a-4e31-8d71-869db8212330\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/04b4ca06-437a-4e31-8d71-869db8212330\",\n \"name\": \"04b4ca06-437a-4e31-8d71-869db8212330\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:35:24.727596551Z\",\n \"endTime\": \"2026-09-29T03:59:24.666199081Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4docerqoft14tulvhsqagod093vb4h still exists (deletion dispatched at 2026-09-29T03:35:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c0082d88e21a4703f8364e810598eaef468c8f03a8b233cc020742561757a3d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"e2e","signature_id":"94b9193d5e4c58e9ab25995cf98d7e40f59dd02fce0847a64eeddcac41395ed7","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000838160\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-jcx588vhqlvr\" to finish creating: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14c6b69c-2c34-4e20-ac15-13d1247f8e46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14c6b69c-2c34-4e20-ac15-13d1247f8e46\",\n \"name\": \"14c6b69c-2c34-4e20-ac15-13d1247f8e46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:05.293809431Z\",\n \"endTime\": \"2026-09-29T03:29:10.066539649Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 21m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000838160\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-jcx588vhqlvr\" to finish creating: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14c6b69c-2c34-4e20-ac15-13d1247f8e46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/14c6b69c-2c34-4e20-ac15-13d1247f8e46\",\n \"name\": \"14c6b69c-2c34-4e20-ac15-13d1247f8e46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:05.293809431Z\",\n \"endTime\": \"2026-09-29T03:29:10.066539649Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 21m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d8c67ac52d45982b07b914f1d40896ba730cf0886309481e36eb0a0552efa425","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"e2e","signature_id":"66d6be4a031933581aba8c93a0a15bbb13e43df2ee76923c1a3675de9e560018","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002990330\u003e: \n resource group \"private-keyvault-l2q7q6dfjh6g\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-l2q7q6dfjh6g\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/964165dd-e4d9-4af4-8298-2f0cb2afadf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/964165dd-e4d9-4af4-8298-2f0cb2afadf3\",\n \"name\": \"964165dd-e4d9-4af4-8298-2f0cb2afadf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:26:33.2115162Z\",\n \"endTime\": \"2026-09-29T03:50:39.923597778Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4do1ko245902o36sb31sgeglbeq2vk still exists (deletion dispatched at 2026-09-29T03:26:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002990330\u003e: \n resource group \"private-keyvault-l2q7q6dfjh6g\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-l2q7q6dfjh6g\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/964165dd-e4d9-4af4-8298-2f0cb2afadf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/964165dd-e4d9-4af4-8298-2f0cb2afadf3\",\n \"name\": \"964165dd-e4d9-4af4-8298-2f0cb2afadf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:26:33.2115162Z\",\n \"endTime\": \"2026-09-29T03:50:39.923597778Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4do1ko245902o36sb31sgeglbeq2vk still exists (deletion dispatched at 2026-09-29T03:26:34Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e4937caed3b515ba555fde358f4bbc7a9d56c5cb9b862ed97f60809d78ef68fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"e2e","signature_id":"f82f633c8ad5359b91191869cc3f1dbab57afeeb315d80bec246ac52f9a2b181","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00157c360\u003e: \n resource group \"rg-candidate-4-23-dz9j6hnkr6jc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-fk9hps\" in resourcegroup=\"rg-candidate-4-23-dz9j6hnkr6jc\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e8320ba1-da57-416f-bcc9-5d820818a38e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e8320ba1-da57-416f-bcc9-5d820818a38e\",\n \"name\": \"e8320ba1-da57-416f-bcc9-5d820818a38e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:27:55.093764971Z\",\n \"endTime\": \"2026-09-29T03:51:59.900618832Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dnb7cc6npj9mnmjas0shvcb5qn63l still exists (deletion dispatched at 2026-09-29T03:27:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00157c360\u003e: \n resource group \"rg-candidate-4-23-dz9j6hnkr6jc\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-fk9hps\" in resourcegroup=\"rg-candidate-4-23-dz9j6hnkr6jc\" to finish deleting: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e8320ba1-da57-416f-bcc9-5d820818a38e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e8320ba1-da57-416f-bcc9-5d820818a38e\",\n \"name\": \"e8320ba1-da57-416f-bcc9-5d820818a38e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:27:55.093764971Z\",\n \"endTime\": \"2026-09-29T03:51:59.900618832Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dnb7cc6npj9mnmjas0shvcb5qn63l still exists (deletion dispatched at 2026-09-29T03:27:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f27d6f58ad38efada43bcf4c9c41ddd931c7852c9cd1124968860722baaa1c33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104751945536245760","occurred_at":"2026-09-29T01:55:42Z","failed_at":"e2e","signature_id":"49163bb2062434c1f083547612fa1c43e90193c5193f060f5e754ecd13438cec","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--vbl2fffvpm77/cluster-hs-pre-6h5wx2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa6f80\u003e: \n failed to create HCP cluster cluster-hs-pre-6h5wx2: failed waiting for cluster=\"cluster-hs-pre-6h5wx2\" in resourcegroup=\"rg-hs-pre--vbl2fffvpm77\" to finish creating: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed26df33-28d6-48f7-aad8-139388423746\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed26df33-28d6-48f7-aad8-139388423746\",\n \"name\": \"ed26df33-28d6-48f7-aad8-139388423746\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:05.091215108Z\",\n \"endTime\": \"2026-09-29T03:29:10.065852363Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--vbl2fffvpm77/cluster-hs-pre-6h5wx2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa6f80\u003e: \n failed to create HCP cluster cluster-hs-pre-6h5wx2: failed waiting for cluster=\"cluster-hs-pre-6h5wx2\" in resourcegroup=\"rg-hs-pre--vbl2fffvpm77\" to finish creating: GET https://rp.j6245760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed26df33-28d6-48f7-aad8-139388423746\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ed26df33-28d6-48f7-aad8-139388423746\",\n \"name\": \"ed26df33-28d6-48f7-aad8-139388423746\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:05.091215108Z\",\n \"endTime\": \"2026-09-29T03:29:10.065852363Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa7e67c7a590a1275d29fc48bfa890bb54cd4ebb44d206ef3a2ff048e5a51837","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","aa7e67c7a590a1275d29fc48bfa890bb54cd4ebb44d206ef3a2ff048e5a51837","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104749868172972032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7200,"pr_state":"open","pr_sha":"4d8592f59c101f4b9b283ce8bd3872f0e2ef7594","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T01:47:21Z","started_at":"2026-09-29T01:47:21Z","completed_at":"2026-09-29T04:24:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:27:24.393997941Z","timing_metadata_checked_at":"2026-09-29T04:27:24.393997941Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:27:24.396581533Z","region_metadata_checked_at":"2026-09-29T04:27:24.396581533Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b9aa7fb78a1e1d7e3cb3c428aa5c715b0ed6811f038859d5927e753d8787d8fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104749868172972032","occurred_at":"2026-09-29T01:47:21Z","failed_at":"e2e","signature_id":"48a25d611124959a4e4e873c795ebd82af1d12806c242e2f13414987c800a115","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:154]: failed to create customer resources for third cluster \"basic-hcp-cluster3\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088a640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-xlptvl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-xlptvl/operationStatuses/08584109556974857386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-lxcj7rrjfmg7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-03-31/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:154]: failed to create customer resources for third cluster \"basic-hcp-cluster3\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088a640\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-xlptvl\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-xlptvl/operationStatuses/08584109556974857386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-lxcj7rrjfmg7/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-03-31/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7224,"pr_state":"closed","pr_sha":"a6cabcda34fdee6066dd2b496192fcb45072451a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T01:37:45Z","started_at":"2026-09-29T01:37:45Z","completed_at":"2026-09-29T04:12:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:17:04.617137398Z","timing_metadata_checked_at":"2026-09-29T04:17:04.617137398Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:17:04.619818288Z","region_metadata_checked_at":"2026-09-29T04:17:04.619818288Z"},"failed_at":["e2e"],"failed_test_count":9,"occurrences":[{"row_id":"14e00c2fa3144f6f1826c7fe427f58a8a9337fac4b5cc6c0c186613447e06548","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","occurred_at":"2026-09-29T01:37:45Z","failed_at":"e2e","signature_id":"5798c3780cd442d290c5794b4c211d8862f5def573fb6e601b9125c130a43dc3","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:293]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001486340\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-wzv4bm-g699tf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-wzv4bm-g699tf/operationStatuses/08584109554270576231\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:293]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001486340\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-noedge-wzv4bm-g699tf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-noedge-wzv4bm-g699tf/operationStatuses/08584109554270576231\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-00-09/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"151e46adf720f7cdfc8873781f04bc404218a0ff164fbfdd40e27ab6a49a5320","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","occurred_at":"2026-09-29T01:37:45Z","failed_at":"e2e","signature_id":"ebf6c103aaa1ca5186dde5ce73213377ebf49f8c057c025b4767b831827e388b","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e962c0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8rl27d\" in resourcegroup=\"rg-np-version-upgrade-8rl27d-cds8pn7rp5pj\" to finish creating: GET https://rp.j0435840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99532f4a-c494-477c-9235-28ce3b3dccdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99532f4a-c494-477c-9235-28ce3b3dccdb\",\n \"name\": \"99532f4a-c494-477c-9235-28ce3b3dccdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:59:24.07754651Z\",\n \"endTime\": \"2026-09-29T03:00:05.923503248Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-21 with version 4.21.34\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e962c0\u003e: \n failed to create NodePool npupgrade-4-21: failed waiting for nodepool=\"npupgrade-4-21\" for cluster \"np-version-upgrade-cluster-8rl27d\" in resourcegroup=\"rg-np-version-upgrade-8rl27d-cds8pn7rp5pj\" to finish creating: GET https://rp.j0435840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99532f4a-c494-477c-9235-28ce3b3dccdb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99532f4a-c494-477c-9235-28ce3b3dccdb\",\n \"name\": \"99532f4a-c494-477c-9235-28ce3b3dccdb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:59:24.07754651Z\",\n \"endTime\": \"2026-09-29T03:00:05.923503248Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"21ec73cb60e446bb73b47240826172c59c03bb2be429121dcda8ba5ea5d23ae3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","occurred_at":"2026-09-29T01:37:45Z","failed_at":"e2e","signature_id":"07dd6cb2c731798c1ba91451e4f4e831a6474bade7ac64e105f54eddf1f77603","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a02cc0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-7gnm6qbmvpcb\" to finish creating: GET https://rp.j0435840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8167fae0-e6eb-4298-bee1-fee2d504da77\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8167fae0-e6eb-4298-bee1-fee2d504da77\",\n \"name\": \"8167fae0-e6eb-4298-bee1-fee2d504da77\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:58:54.33539143Z\",\n \"endTime\": \"2026-09-29T02:59:36.376050844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a02cc0\u003e: \n failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-7gnm6qbmvpcb\" to finish creating: GET https://rp.j0435840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8167fae0-e6eb-4298-bee1-fee2d504da77\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8167fae0-e6eb-4298-bee1-fee2d504da77\",\n \"name\": \"8167fae0-e6eb-4298-bee1-fee2d504da77\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:58:54.33539143Z\",\n \"endTime\": \"2026-09-29T02:59:36.376050844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2440c7713ccebe5d66ca3433ac24d704c0bd0f68c3e6e860a6f1d88d64e6b91e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","occurred_at":"2026-09-29T01:37:45Z","failed_at":"e2e","signature_id":"7cb246d0dd4b1d5c0d9504db66386b1d3fc92b84852323cf24324f2b2ca3ca03","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011bf9c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-2plrgsl5fwgr\" to finish creating: GET https://rp.j0435840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/476732bf-65f0-4455-afd3-17ce246aa2ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/476732bf-65f0-4455-afd3-17ce246aa2ca\",\n \"name\": \"476732bf-65f0-4455-afd3-17ce246aa2ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:32.197258881Z\",\n \"endTime\": \"2026-09-29T03:01:14.992686131Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011bf9c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-2plrgsl5fwgr\" to finish creating: GET https://rp.j0435840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/476732bf-65f0-4455-afd3-17ce246aa2ca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/476732bf-65f0-4455-afd3-17ce246aa2ca\",\n \"name\": \"476732bf-65f0-4455-afd3-17ce246aa2ca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:32.197258881Z\",\n \"endTime\": \"2026-09-29T03:01:14.992686131Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4985ca1b7f9ce11f1cdb27ee75d3b91c494dc672480f9be4d7527b4fcc477d0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","occurred_at":"2026-09-29T01:37:45Z","failed_at":"e2e","signature_id":"d89f25ae6ba4256a89a04462f3bd4ee5766c5a9a38461d72b34e552889ad9eab","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:572]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001101800\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-t7zv7w-tcz45g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-t7zv7w-tcz45g/operationStatuses/08584109555384255677\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-00-44/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-t7zv7w-lnxhzdqzcz5n/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-caqbep47wy2o6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:572]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001101800\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-downgrade-t7zv7w-tcz45g\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-downgrade-t7zv7w-tcz45g/operationStatuses/08584109555384255677\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-00-44/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-t7zv7w-lnxhzdqzcz5n/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-caqbep47wy2o6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"872b44e50f9f5b8b6674e7e23ecee2e040b3709b7d1cc9f3ee0de3177d6ffc9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","occurred_at":"2026-09-29T01:37:45Z","failed_at":"e2e","signature_id":"d41717e9a6df5345080bc98ba130e5a10e636e0a3c233328019006e1283f5d3f","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:348]: failed to grant AcrPull to second MI on ACR\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001023830\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-92f84hwfsjvp/providers/Microsoft.ContainerRegistry/registries/acrpullcreate92f84hwfsjvp/providers/Microsoft.Authorization/roleAssignments/500314c4-ec16-5346-bdf6-ef93cc9ce507\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: RoleAssignmentLimitExceeded\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:348]: failed to grant AcrPull to second MI on ACR\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001023830\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/acr-pull-create-92f84hwfsjvp/providers/Microsoft.ContainerRegistry/registries/acrpullcreate92f84hwfsjvp/providers/Microsoft.Authorization/roleAssignments/500314c4-ec16-5346-bdf6-ef93cc9ce507\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: RoleAssignmentLimitExceeded\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a6e506f0858fbc33e1828729abe30ab0b12b4ac3a77c6f1e16b5e112b3c1573d","failure_pattern":"ERROR CODE: RoleAssignmentLimitExceeded; provider Microsoft.Authorization","search_query":"ERROR CODE: RoleAssignmentLimitExceeded"}},{"row_id":"a2f6a73d22b979942d12b2588a4ccd1a6dc3b10b64cb6bf22a4cf1debc03e921","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","occurred_at":"2026-09-29T01:37:45Z","failed_at":"e2e","signature_id":"f28b26d4aa76fff53f003758b1a41bfccea4cd0caf5af76a8621aa82007dfbb2","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:248]: Told to stop trying after 15.612s.\nDNS Zone Contributor role assignment should succeed once principal propagates\nassigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud to a626cc91-d8ba-4fcd-b448-1c0201b41a1e: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/b39c004e-40fa-570e-813c-44e71fc436b7\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: RoleAssignmentLimitExceeded\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n}\n--------------------------------------------------------------------------------\n: assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud to a626cc91-d8ba-4fcd-b448-1c0201b41a1e: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/b39c004e-40fa-570e-813c-44e71fc436b7\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: RoleAssignmentLimitExceeded\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n}\n--------------------------------------------------------------------------------\n\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000e58700\u003e: \n assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud to a626cc91-d8ba-4fcd-b448-1c0201b41a1e: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/b39c004e-40fa-570e-813c-44e71fc436b7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: PrincipalNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"PrincipalNotFound\",\n \"message\": \"Principal a626cc91d8ba4fcdb4481c0201b41a1e does not exist in the directory XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX. Check that you have the correct principal ID. If you are creating this principal and then immediately assigning a role, this error might be related to a replication delay. In this case, set the role assignment principalType property to a value, such as ServicePrincipal, User, or Group. See https://aka.ms/docs-principaltype\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:248]: Told to stop trying after 15.612s.\nDNS Zone Contributor role assignment should succeed once principal propagates\nassigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud to a626cc91-d8ba-4fcd-b448-1c0201b41a1e: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/b39c004e-40fa-570e-813c-44e71fc436b7\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: RoleAssignmentLimitExceeded\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n}\n--------------------------------------------------------------------------------\n: assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud to a626cc91-d8ba-4fcd-b448-1c0201b41a1e: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/b39c004e-40fa-570e-813c-44e71fc436b7\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: RoleAssignmentLimitExceeded\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n}\n--------------------------------------------------------------------------------\n\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000e58700\u003e: \n assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud to a626cc91-d8ba-4fcd-b448-1c0201b41a1e: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.8h5b.j0435840.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/b39c004e-40fa-570e-813c-44e71fc436b7\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: PrincipalNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"PrincipalNotFound\",\n \"message\": \"Principal a626cc91d8ba4fcdb4481c0201b41a1e does not exist in the directory XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX. Check that you have the correct principal ID. If you are creating this principal and then immediately assigning a role, this error might be related to a replication delay. In this case, set the role assignment principalType property to a value, such as ServicePrincipal, User, or Group. See https://aka.ms/docs-principaltype\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ceadf43459c98a2e2f9b61df48bfd6a5c28973cefdcf40c376ce51ca24ebe4c","failure_pattern":"assigning role \u003cuuid\u003e on /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft","search_query":"assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft"}},{"row_id":"d39f641f63006858e5e3539da68764668480550d09cd685f0af7c66139b9c23d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","occurred_at":"2026-09-29T01:37:45Z","failed_at":"e2e","signature_id":"27eac4629b31de57b043ae1a8339716d0401bfae44f7f647a4ef7a218870f059","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:712]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001744a60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-ndgcc2-tplzd9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-ndgcc2-tplzd9/operationStatuses/08584109554179204837\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-ndgcc2-kcw2dvn877rh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-bmtu7bm547ml6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:712]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001744a60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-dg-minor-ndgcc2-tplzd9\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-dg-minor-ndgcc2-tplzd9/operationStatuses/08584109554179204837\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-ndgcc2-kcw2dvn877rh/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-bmtu7bm547ml6\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"da45ba48dbe41355317a81dd37d7f8156b1225e77779bb0b0ec2997674918af2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747427830435840","occurred_at":"2026-09-29T01:37:45Z","failed_at":"e2e","signature_id":"6d77e3b201b4acc09eac2d24cbeb2b32711345b337d53b664e5fb88760fa9952","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000907f00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-hz86tzsmqsq2\" to finish creating: GET https://rp.j0435840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f13c2d35-28fa-4d98-8293-daf234841872\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f13c2d35-28fa-4d98-8293-daf234841872\",\n \"name\": \"f13c2d35-28fa-4d98-8293-daf234841872\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:59:48.054303721Z\",\n \"endTime\": \"2026-09-29T03:18:48.042342466Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e8n2j1e5m4m0t8i-np-1-7tk64-xgmkn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hz86tzsmqsq2--managed/providers/Microsoft.Compute/virtualMachines/e8n2j1e5m4m0t8i-np-1-7tk64-xgmkn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e8n2j1e5m4m0t8i-np-1-7tk64-xgmkn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hz86tzsmqsq2--managed/providers/Microsoft.Compute/virtualMachines/e8n2j1e5m4m0t8i-np-1-7tk64-xgmkn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000907f00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-hz86tzsmqsq2\" to finish creating: GET https://rp.j0435840.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f13c2d35-28fa-4d98-8293-daf234841872\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f13c2d35-28fa-4d98-8293-daf234841872\",\n \"name\": \"f13c2d35-28fa-4d98-8293-daf234841872\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:59:48.054303721Z\",\n \"endTime\": \"2026-09-29T03:18:48.042342466Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e8n2j1e5m4m0t8i-np-1-7tk64-xgmkn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hz86tzsmqsq2--managed/providers/Microsoft.Compute/virtualMachines/e8n2j1e5m4m0t8i-np-1-7tk64-xgmkn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e8n2j1e5m4m0t8i-np-1-7tk64-xgmkn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-hz86tzsmqsq2--managed/providers/Microsoft.Compute/virtualMachines/e8n2j1e5m4m0t8i-np-1-7tk64-xgmkn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["0ceadf43459c98a2e2f9b61df48bfd6a5c28973cefdcf40c376ce51ca24ebe4c","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","a6e506f0858fbc33e1828729abe30ab0b12b4ac3a77c6f1e16b5e112b3c1573d"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747414274445312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T01:37:42Z","started_at":"2026-09-29T01:37:42Z","completed_at":"2026-09-29T04:05:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:17:04.567738476Z","timing_metadata_checked_at":"2026-09-29T04:17:04.567738476Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:17:04.570553266Z","region_metadata_checked_at":"2026-09-29T04:17:04.570553266Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747399384666112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"e7500a3658c1692b6c32705cb7edf403b704aae4","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T01:37:38Z","started_at":"2026-09-29T01:37:38Z","completed_at":"2026-09-29T04:29:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:37:32.960348082Z","timing_metadata_checked_at":"2026-09-29T04:37:32.960348082Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:37:32.964509068Z","region_metadata_checked_at":"2026-09-29T04:37:32.964509068Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b4513d786be4ca6c3b8132124c405b17c26aee0c85889c1aa68e6ac4da9a5719","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747399384666112","occurred_at":"2026-09-29T01:37:38Z","failed_at":"alert","signature_id":"bfa1abd3d3de88f4b11ae3a45ccac07507cd499b0b8195856fb93ae774eb5cf9","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T03:06:32Z\n Ended: 2026-09-29T03:12:31Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4666112-svc\", component=\"backend\", name=\"clustervalidationcontrolplaneidentitiespermissionsclustervalidation\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clustervalidationcontrolplaneidentitiespermissionsclustervalidation has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747373145100288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7221,"pr_state":"closed","pr_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","final_merged_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T01:37:32Z","started_at":"2026-09-29T01:37:32Z","completed_at":"2026-09-29T04:16:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:27:24.871319125Z","timing_metadata_checked_at":"2026-09-29T04:27:24.871319125Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:27:24.873848917Z","region_metadata_checked_at":"2026-09-29T04:27:24.873848917Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"58d1a5761742a895c46faeb74723cb489541411757e4fcfe607cac88a7c9f91b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747373145100288","occurred_at":"2026-09-29T01:37:32Z","failed_at":"e2e","signature_id":"14041b6afd873bee57610a7594c31f9381a85c761ee8df6fe5faf0b023b6ae79","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d57340\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-4hw9tp\" in resourcegroup=\"rg-np-noedge-4hw9tp-75mk89p5kg87\" to finish creating: GET https://rp.j5100288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09884210-bfbb-47ea-9183-54386fd0d07e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09884210-bfbb-47ea-9183-54386fd0d07e\",\n \"name\": \"09884210-bfbb-47ea-9183-54386fd0d07e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:17:00.285867065Z\",\n \"endTime\": \"2026-09-29T03:36:07.78102623Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a1i2b3l9p4a9q2d-npnoedge-4-20-nqw8g-sdr9l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4hw9tp-75mk89p5kg87-np-ne-4hw9tp--managed/providers/Microsoft.Compute/virtualMachines/a1i2b3l9p4a9q2d-npnoedge-4-20-nqw8g-sdr9l to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a1i2b3l9p4a9q2d-npnoedge-4-20-nqw8g-sdr9l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4hw9tp-75mk89p5kg87-np-ne-4hw9tp--managed/providers/Microsoft.Compute/virtualMachines/a1i2b3l9p4a9q2d-npnoedge-4-20-nqw8g-sdr9l to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d57340\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-4hw9tp\" in resourcegroup=\"rg-np-noedge-4hw9tp-75mk89p5kg87\" to finish creating: GET https://rp.j5100288.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09884210-bfbb-47ea-9183-54386fd0d07e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/09884210-bfbb-47ea-9183-54386fd0d07e\",\n \"name\": \"09884210-bfbb-47ea-9183-54386fd0d07e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:17:00.285867065Z\",\n \"endTime\": \"2026-09-29T03:36:07.78102623Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a1i2b3l9p4a9q2d-npnoedge-4-20-nqw8g-sdr9l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4hw9tp-75mk89p5kg87-np-ne-4hw9tp--managed/providers/Microsoft.Compute/virtualMachines/a1i2b3l9p4a9q2d-npnoedge-4-20-nqw8g-sdr9l to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a1i2b3l9p4a9q2d-npnoedge-4-20-nqw8g-sdr9l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-4hw9tp-75mk89p5kg87-np-ne-4hw9tp--managed/providers/Microsoft.Compute/virtualMachines/a1i2b3l9p4a9q2d-npnoedge-4-20-nqw8g-sdr9l to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c88af5b0827b55c701d4f4f9da4620a28837a194e2c3b1133f68f42db4ce4c33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747373145100288","occurred_at":"2026-09-29T01:37:32Z","failed_at":"e2e","signature_id":"7e64b242e8fc685b3f5f052bb0fd9c995248e24a651a7e09199ecbdcedc3d0e4","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00308adb0\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-f5zptcd4xflk\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-f5zptcd4xflk, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00308adb0\u003e: \n failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-f5zptcd4xflk\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-f5zptcd4xflk, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-sbrs7l4xp4mk\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-sbrs7l4xp4mk, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747357269659648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"closed","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T01:37:28Z","started_at":"2026-09-29T01:37:28Z","completed_at":"2026-09-29T04:06:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:17:04.545192558Z","timing_metadata_checked_at":"2026-09-29T04:17:04.545192558Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:17:04.548704845Z","region_metadata_checked_at":"2026-09-29T04:17:04.548704845Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e2b6edbeabda1849f78b3dd2b586948613be6a00c67c336ffd439ab5b1119936","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747357269659648","occurred_at":"2026-09-29T01:37:28Z","failed_at":"e2e","signature_id":"add9f7d044b7a7e78e352e938182326308450c36721b815567cde57573e65d59","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc0013f83f0\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47km4.apps.aro.z8j4o9t8b9u3i6w.b8n7.j9659648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47km4.apps.aro.z8j4o9t8b9u3i6w.b8n7.j9659648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47km4.apps.aro.z8j4o9t8b9u3i6w.b8n7.j9659648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:203]: failed to verify cluster workloads and pods after autoscaling update\nUnexpected error:\n \u003c*errors.joinError | 0xc0013f83f0\u003e: \n VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47km4.apps.aro.z8j4o9t8b9u3i6w.b8n7.j9659648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47km4.apps.aro.z8j4o9t8b9u3i6w.b8n7.j9659648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47km4.apps.aro.z8j4o9t8b9u3i6w.b8n7.j9659648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-47km4.apps.aro.z8j4o9t8b9u3i6w.b8n7.j9659648.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-47km4.apps.aro.z8j4o9t8b9u3i6w.b8n7.j9659648.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-47km4.apps.aro.z8j4o9t8b9u3i6w.b8n7.j9659648.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["aaca9380cc770ec74652811b10adcc869ebd3b578ff2c4827ab22a70f3f234b3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747348675530752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T01:37:26Z","started_at":"2026-09-29T01:37:26Z","completed_at":"2026-09-29T04:26:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:37:33.111691971Z","timing_metadata_checked_at":"2026-09-29T04:37:33.111691971Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:37:33.114417762Z","region_metadata_checked_at":"2026-09-29T04:37:33.114417762Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"77f1f83ebbb98ef438ab6eb39c7b3f8fdd971cc7b7a6db81731c2c5ebfafafc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747348675530752","occurred_at":"2026-09-29T01:37:26Z","failed_at":"e2e","signature_id":"4f674f930f156c4c656fe9893dd51ff3fd91d417532ebf50d7372800d72bfde1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pxxjsb9mbpw2/cluster-candidate-4-22-vchxk6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c28520\u003e: \n failed to create HCP cluster cluster-candidate-4-22-vchxk6: failed waiting for cluster=\"cluster-candidate-4-22-vchxk6\" in resourcegroup=\"rg-candidate-4-22-pxxjsb9mbpw2\" to finish creating: GET https://rp.j5530752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dcd43e84-3bac-4e4c-a704-8ad21694f9d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dcd43e84-3bac-4e4c-a704-8ad21694f9d5\",\n \"name\": \"dcd43e84-3bac-4e4c-a704-8ad21694f9d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:51:39.168217674Z\",\n \"endTime\": \"2026-09-29T03:20:45.495027988Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator, cluster-network-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-pxxjsb9mbpw2/cluster-candidate-4-22-vchxk6 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c28520\u003e: \n failed to create HCP cluster cluster-candidate-4-22-vchxk6: failed waiting for cluster=\"cluster-candidate-4-22-vchxk6\" in resourcegroup=\"rg-candidate-4-22-pxxjsb9mbpw2\" to finish creating: GET https://rp.j5530752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dcd43e84-3bac-4e4c-a704-8ad21694f9d5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dcd43e84-3bac-4e4c-a704-8ad21694f9d5\",\n \"name\": \"dcd43e84-3bac-4e4c-a704-8ad21694f9d5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:51:39.168217674Z\",\n \"endTime\": \"2026-09-29T03:20:45.495027988Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-image-registry-operator, ingress-operator, cluster-network-operator, csi-snapshot-controller-operator; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-policy-controller deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c355edbf3646651e3e1bf25edd2f7bc1a59a3126c247b9fabef0170029fd43a5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [cluster-image-registry-operator, cluster-network-operator, cluster-policy-controller, csi-snapshot-controller-operator, ingress-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b215bd4a59f397175806266adacfe2b8d1b4257e5db1c7e39c4b8018962b1dc8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747348675530752","occurred_at":"2026-09-29T01:37:26Z","failed_at":"e2e","signature_id":"60f169b316d674b6d3f83f4adf0e7bc4893ca42ccd88ba82adf6a65c465b5d84","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00075e620\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-lzq55sh9jmq4\" to finish creating: GET https://rp.j5530752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/772d74f8-28ec-4061-981a-203cb9d2341a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/772d74f8-28ec-4061-981a-203cb9d2341a\",\n \"name\": \"772d74f8-28ec-4061-981a-203cb9d2341a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:53:30.135245133Z\",\n \"endTime\": \"2026-09-29T03:22:32.693389266Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 5m0s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00075e620\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-lzq55sh9jmq4\" to finish creating: GET https://rp.j5530752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/772d74f8-28ec-4061-981a-203cb9d2341a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/772d74f8-28ec-4061-981a-203cb9d2341a\",\n \"name\": \"772d74f8-28ec-4061-981a-203cb9d2341a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:53:30.135245133Z\",\n \"endTime\": \"2026-09-29T03:22:32.693389266Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 5m0s ago; hosted cluster degraded: UnavailableReplicas: kube-controller-manager deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6b9eb312202879fb9a6f390df391c95e64889c51047a74b640051be8935c181f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-controller-manager]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e7d8c731ae2770acc8c89c789de9c4ea9af900925902c20bfc64811eb5157269","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747348675530752","occurred_at":"2026-09-29T01:37:26Z","failed_at":"e2e","signature_id":"517ada13c5bf432c109f2ae8387cc4b3e96d1db9062765e62f73378b0a367478","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:154]: failed to create customer resources for third cluster \"basic-hcp-cluster3\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e440\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-zwklrj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-zwklrj/operationStatuses/08584109554178456377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-04-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:154]: failed to create customer resources for third cluster \"basic-hcp-cluster3\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e440\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-zwklrj\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-zwklrj/operationStatuses/08584109554178456377\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-04-11/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["6b9eb312202879fb9a6f390df391c95e64889c51047a74b640051be8935c181f","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","c355edbf3646651e3e1bf25edd2f7bc1a59a3126c247b9fabef0170029fd43a5"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747333580230656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7215,"pr_state":"open","pr_sha":"4c3b3d5d645acb69fa7458f18c9319f14d4142d1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T01:37:23Z","started_at":"2026-09-29T01:37:23Z","completed_at":"2026-09-29T04:24:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:27:24.695612283Z","timing_metadata_checked_at":"2026-09-29T04:27:24.695612283Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:27:24.698815673Z","region_metadata_checked_at":"2026-09-29T04:27:24.698815673Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"1019f9d452a1343fe347bbf0c245afece49b2551c1912d7fbf3ef4a01f0e2045","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747333580230656","occurred_at":"2026-09-29T01:37:23Z","failed_at":"e2e","signature_id":"4a9c1af93ff08831bd52d7ab92fcb3e7359ed80a48e38ee990d82b3ffef5372a","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0019a67e0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-sbrs7l4xp4mk\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-sbrs7l4xp4mk, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0019a67e0\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-sbrs7l4xp4mk\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-sbrs7l4xp4mk, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-sbrs7l4xp4mk\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-sbrs7l4xp4mk, error: context deadline exceeded"}},{"row_id":"1f42689e277db81ff85741662f559d604bf923b337118220504f30b9ffcfcd87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747333580230656","occurred_at":"2026-09-29T01:37:23Z","failed_at":"e2e","signature_id":"4861cdacb696d2101485082fc0296f581b3d84df1e60c0eef44066fadfda980d","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:248]: Told to stop trying after 15.676s.\nDNS Zone Contributor role assignment should succeed once principal propagates\nassigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud to c1778d09-4e30-43d3-a6e5-c12816d71c41: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/18eec22c-5874-5077-bd44-80689d165720\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: RoleAssignmentLimitExceeded\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n}\n--------------------------------------------------------------------------------\n: assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud to c1778d09-4e30-43d3-a6e5-c12816d71c41: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/18eec22c-5874-5077-bd44-80689d165720\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: RoleAssignmentLimitExceeded\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n}\n--------------------------------------------------------------------------------\n\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000ebe2a0\u003e: \n assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud to c1778d09-4e30-43d3-a6e5-c12816d71c41: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/18eec22c-5874-5077-bd44-80689d165720\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: PrincipalNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"PrincipalNotFound\",\n \"message\": \"Principal c1778d094e3043d3a6e5c12816d71c41 does not exist in the directory XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX. Check that you have the correct principal ID. If you are creating this principal and then immediately assigning a role, this error might be related to a replication delay. In this case, set the role assignment principalType property to a value, such as ServicePrincipal, User, or Group. See https://aka.ms/docs-principaltype\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:248]: Told to stop trying after 15.676s.\nDNS Zone Contributor role assignment should succeed once principal propagates\nassigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud to c1778d09-4e30-43d3-a6e5-c12816d71c41: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/18eec22c-5874-5077-bd44-80689d165720\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: RoleAssignmentLimitExceeded\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n}\n--------------------------------------------------------------------------------\n: assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud to c1778d09-4e30-43d3-a6e5-c12816d71c41: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/18eec22c-5874-5077-bd44-80689d165720\n--------------------------------------------------------------------------------\nRESPONSE 400: 400 Bad Request\nERROR CODE: RoleAssignmentLimitExceeded\n--------------------------------------------------------------------------------\n{\n \"error\": {\n \"code\": \"RoleAssignmentLimitExceeded\",\n \"message\": \"No more role assignments can be created.\"\n }\n}\n--------------------------------------------------------------------------------\n\nAt one point, however, the function did return successfully.\nYet, Eventually failed because the matcher was not satisfied:\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000ebe2a0\u003e: \n assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud to c1778d09-4e30-43d3-a6e5-c12816d71c41: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-rpc5rk6cx9j4--managed/providers/Microsoft.Network/dnsZones/aro.ingress-cert-le.9hzx.j0230656.hcp.osadev.cloud/providers/Microsoft.Authorization/roleAssignments/18eec22c-5874-5077-bd44-80689d165720\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: PrincipalNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"PrincipalNotFound\",\n \"message\": \"Principal c1778d094e3043d3a6e5c12816d71c41 does not exist in the directory XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX. Check that you have the correct principal ID. If you are creating this principal and then immediately assigning a role, this error might be related to a replication delay. In this case, set the role assignment principalType property to a value, such as ServicePrincipal, User, or Group. See https://aka.ms/docs-principaltype\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ceadf43459c98a2e2f9b61df48bfd6a5c28973cefdcf40c376ce51ca24ebe4c","failure_pattern":"assigning role \u003cuuid\u003e on /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft","search_query":"assigning role befefa01-2a29-4197-83a8-272ff33ce314 on /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-c7hffp5rx8qh--managed/providers/Microsoft"}},{"row_id":"3ceaacda17b8f073f3f63e0766b5458f0a83437db735631264457ed369c0e5bc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747333580230656","occurred_at":"2026-09-29T01:37:23Z","failed_at":"e2e","signature_id":"155908f93ab9d60173f27a3b80d860cdb60ebc9a64828336c942df149be1a6ec","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011b6150\u003e: \n resource group \"patch-tags-vb572dcgrtxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vb572dcgrtxw\" to finish deleting: GET https://rp.j0230656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f22ac5-a416-4ae3-a322-aefdc0c7dba6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f22ac5-a416-4ae3-a322-aefdc0c7dba6\",\n \"name\": \"03f22ac5-a416-4ae3-a322-aefdc0c7dba6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:11.459457944Z\",\n \"endTime\": \"2026-09-29T03:24:16.961720696Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dh8v1rajainbkav3mf373qfaraar5 still exists (deletion dispatched at 2026-09-29T03:00:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011b6150\u003e: \n resource group \"patch-tags-vb572dcgrtxw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-vb572dcgrtxw\" to finish deleting: GET https://rp.j0230656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f22ac5-a416-4ae3-a322-aefdc0c7dba6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f22ac5-a416-4ae3-a322-aefdc0c7dba6\",\n \"name\": \"03f22ac5-a416-4ae3-a322-aefdc0c7dba6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:11.459457944Z\",\n \"endTime\": \"2026-09-29T03:24:16.961720696Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t4dh8v1rajainbkav3mf373qfaraar5 still exists (deletion dispatched at 2026-09-29T03:00:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8d3400a05e2499d1a34a70da127b59756c75ee5d9fdc70f75aea055c606d70b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747333580230656","occurred_at":"2026-09-29T01:37:23Z","failed_at":"e2e","signature_id":"1661f171ed1243433a8cb435d46447de028f234837fe682dde1055e2b6793c73","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:154]: failed to create customer resources for third cluster \"basic-hcp-cluster3\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9e5c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-2f7rt6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-2f7rt6/operationStatuses/08584109557393570032\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-01-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-vbxtq22792db/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:154]: failed to create customer resources for third cluster \"basic-hcp-cluster3\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f9e5c0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-basic-hcp-cluster3-2f7rt6\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-basic-hcp-cluster3-2f7rt6/operationStatuses/08584109557393570032\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard0-01-15/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n },\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/customer-rg-vbxtq22792db/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-igpwm4vimfr3y\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed"}},{"row_id":"9f3f330fd30e0b3386eb1c67357ac08648a02c75a9c2b9e235ac167fc5fbf7f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747333580230656","occurred_at":"2026-09-29T01:37:23Z","failed_at":"e2e","signature_id":"81d270f668c60015346b1a75b5f5d3e295051ce69e6c26c6977feeb39a101728","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001363e60\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ls92nv\" in resourcegroup=\"rg-np-version-upgrade-ls92nv-m7plkttwtb8b\" to finish creating: GET https://rp.j0230656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37b955b4-cd1d-4d8f-b786-4c0e4e83684b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37b955b4-cd1d-4d8f-b786-4c0e4e83684b\",\n \"name\": \"37b955b4-cd1d-4d8f-b786-4c0e4e83684b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:01:04.522235796Z\",\n \"endTime\": \"2026-09-29T03:01:49.077146384Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001363e60\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ls92nv\" in resourcegroup=\"rg-np-version-upgrade-ls92nv-m7plkttwtb8b\" to finish creating: GET https://rp.j0230656.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37b955b4-cd1d-4d8f-b786-4c0e4e83684b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/37b955b4-cd1d-4d8f-b786-4c0e4e83684b\",\n \"name\": \"37b955b4-cd1d-4d8f-b786-4c0e4e83684b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:01:04.522235796Z\",\n \"endTime\": \"2026-09-29T03:01:49.077146384Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["0ceadf43459c98a2e2f9b61df48bfd6a5c28973cefdcf40c376ce51ca24ebe4c","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","82f810366f876da7711015ea6b8c99b054ca0f8d38feea2e1783563b0dc9ab4d","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747312151531520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7142,"pr_state":"closed","pr_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","final_merged_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T01:37:18Z","started_at":"2026-09-29T01:37:18Z","completed_at":"2026-09-29T04:02:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:06:57.818394154Z","timing_metadata_checked_at":"2026-09-29T04:06:57.818394154Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:06:57.822963939Z","region_metadata_checked_at":"2026-09-29T04:06:57.822963939Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7233/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747306141093888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7233,"pr_state":"open","pr_sha":"b93b8c67088465e0f9e9154753b80f60071d5f91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-29T01:37:16Z","started_at":"2026-09-29T01:37:16Z","completed_at":"2026-09-29T04:29:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:37:33.208637143Z","timing_metadata_checked_at":"2026-09-29T04:37:33.208637143Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:37:33.211314734Z","region_metadata_checked_at":"2026-09-29T04:37:33.211314734Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"17ec6268713231ccd452e114eca9e0881f130199320c8b3a3e7a4f50fedefcef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7233/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747306141093888","occurred_at":"2026-09-29T01:37:16Z","failed_at":"e2e","signature_id":"ef6e3f9265864aa123008ad7fb18cd8d9b650dedbcea2598c3f6f73df198ce9a","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003523480\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5dbmbrljhcdv\" to finish creating: GET https://rp.j1093888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a1400cdf-ff4d-476d-93e5-c28138a36e76\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a1400cdf-ff4d-476d-93e5-c28138a36e76\",\n \"name\": \"a1400cdf-ff4d-476d-93e5-c28138a36e76\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:21:57.572662185Z\",\n \"endTime\": \"2026-09-29T03:40:59.616953542Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5u9e4g3s4k1j8w-np-delete-main-wsv29-j9kzc: Failed: interfaces failed to create or update. err: failed to get existing resource nodepool-delete-5dbmbrljhcdv--managed/e5u9e4g3s4k1j8w-np-delete-main-wsv29-j9kzc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-5dbmbrljhcdv--managed/providers/Microsoft.Network/networkInterfaces/e5u9e4g3s4k1j8w-np-delete-main-wsv29-j9kzc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE UNAVAILABLE\\n--------------------------------------------------------------------------------\\n{\\n \\\"Error\\\": {\\n \\\"Details\\\": [\\n {\\n \\\"Code\\\": \\\"RetryableErrorDueToTooManyCalls\\\",\\n \\\"Message\\\": \\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX was used to perform too many calls within last 5 minutes. The number of calls exceeds Microsoft.Network throttling limit. ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc003523480\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-5dbmbrljhcdv\" to finish creating: GET https://rp.j1093888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a1400cdf-ff4d-476d-93e5-c28138a36e76\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a1400cdf-ff4d-476d-93e5-c28138a36e76\",\n \"name\": \"a1400cdf-ff4d-476d-93e5-c28138a36e76\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:21:57.572662185Z\",\n \"endTime\": \"2026-09-29T03:40:59.616953542Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e5u9e4g3s4k1j8w-np-delete-main-wsv29-j9kzc: Failed: interfaces failed to create or update. err: failed to get existing resource nodepool-delete-5dbmbrljhcdv--managed/e5u9e4g3s4k1j8w-np-delete-main-wsv29-j9kzc-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-5dbmbrljhcdv--managed/providers/Microsoft.Network/networkInterfaces/e5u9e4g3s4k1j8w-np-delete-main-wsv29-j9kzc-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE UNAVAILABLE\\n--------------------------------------------------------------------------------\\n{\\n \\\"Error\\\": {\\n \\\"Details\\\": [\\n {\\n \\\"Code\\\": \\\"RetryableErrorDueToTooManyCalls\\\",\\n \\\"Message\\\": \\\"Subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX was used to perform too many calls within last 5 minutes. The number of calls exceeds Microsoft.Network throttling limit. ... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-mvqbmn55cv5b\" to finish creating: GET https://rp"}},{"row_id":"80cc53aa490969b9a553ec80565062401a8ed1902cc17a3c4c1c4c90431d296a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7233/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747306141093888","occurred_at":"2026-09-29T01:37:16Z","failed_at":"e2e","signature_id":"8dd589f459a86d881b188785f691e9bda4320d0243a193e6e82b67802271d3a5","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-dsj6dg\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc2a60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-dsj6dg: failed waiting for cluster=\"cp-ystream-upgrade-5-0-dsj6dg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-7r7zcv5wwrgs\" to finish creating: GET https://rp.j1093888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffbc98da-1490-42f3-ad07-211f811da461\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffbc98da-1490-42f3-ad07-211f811da461\",\n \"name\": \"ffbc98da-1490-42f3-ad07-211f811da461\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:56:58.872309144Z\",\n \"endTime\": \"2026-09-29T03:25:59.817577076Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-dsj6dg\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc2a60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-5-0-dsj6dg: failed waiting for cluster=\"cp-ystream-upgrade-5-0-dsj6dg\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-7r7zcv5wwrgs\" to finish creating: GET https://rp.j1093888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffbc98da-1490-42f3-ad07-211f811da461\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffbc98da-1490-42f3-ad07-211f811da461\",\n \"name\": \"ffbc98da-1490-42f3-ad07-211f811da461\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:56:58.872309144Z\",\n \"endTime\": \"2026-09-29T03:25:59.817577076Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c39232e1c00a13289b34dc2f012ae9ea8d2cea03800e8094234c06d3b67de93f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7233/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747306141093888","occurred_at":"2026-09-29T01:37:16Z","failed_at":"e2e","signature_id":"acdcdce794d2803efdf3c74af20d863770dadc002c8cc62cbb6d19b37a9109b3","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8aa20\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-ttd7rl8h2rnf\" to finish creating: GET https://rp.j1093888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77a8e181-0244-40af-8d61-cd88694f3a96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77a8e181-0244-40af-8d61-cd88694f3a96\",\n \"name\": \"77a8e181-0244-40af-8d61-cd88694f3a96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:57:01.706166666Z\",\n \"endTime\": \"2026-09-29T03:26:09.704113658Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:89]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f8aa20\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-ttd7rl8h2rnf\" to finish creating: GET https://rp.j1093888.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77a8e181-0244-40af-8d61-cd88694f3a96\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/77a8e181-0244-40af-8d61-cd88694f3a96\",\n \"name\": \"77a8e181-0244-40af-8d61-cd88694f3a96\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:57:01.706166666Z\",\n \"endTime\": \"2026-09-29T03:26:09.704113658Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747285383483392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T01:37:11Z","started_at":"2026-09-29T01:37:11Z","completed_at":"2026-09-29T04:09:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:17:04.595995674Z","timing_metadata_checked_at":"2026-09-29T04:17:04.595995674Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:17:04.598754464Z","region_metadata_checked_at":"2026-09-29T04:17:04.598754464Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1b8e1dc8d7af7c73a34af1b6780fa5d21b8338b64b4901b257c3287d19a944c2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747285383483392","occurred_at":"2026-09-29T01:37:11Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T03:02:24Z\n Ended: 2026-09-29T03:18:23Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3483392-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"9a163a8755f3f6b809ae6763eb42e7a77cf996191785982059a716af0f1f2480","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747285383483392","occurred_at":"2026-09-29T01:37:11Z","failed_at":"e2e","signature_id":"033bd4bed27c6501fcfae7ece88fb5408acbf2f3212a6715eab2644fa1cabe97","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-x756tn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b82d40\u003e: \n DNS for route host agnhost-e2e-sample-app-z4rfg.apps.aro.x8b7b2l9a5c4m4j.y2zb.j3483392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-z4rfg.apps.aro.x8b7b2l9a5c4m4j.y2zb.j3483392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-z4rfg.apps.aro.x8b7b2l9a5c4m4j.y2zb.j3483392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-22-x756tn\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b82d40\u003e: \n DNS for route host agnhost-e2e-sample-app-z4rfg.apps.aro.x8b7b2l9a5c4m4j.y2zb.j3483392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-z4rfg.apps.aro.x8b7b2l9a5c4m4j.y2zb.j3483392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-z4rfg.apps.aro.x8b7b2l9a5c4m4j.y2zb.j3483392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-z4rfg.apps.aro.x8b7b2l9a5c4m4j.y2zb.j3483392.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-z4rfg.apps.aro.x8b7b2l9a5c4m4j.y2zb.j3483392.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-z4rfg.apps.aro.x8b7b2l9a5c4m4j.y2zb.j3483392.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T01:37:08Z","started_at":"2026-09-29T01:37:08Z","completed_at":"2026-09-29T04:24:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:27:24.217337503Z","timing_metadata_checked_at":"2026-09-29T04:27:24.217337503Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:27:24.220622892Z","region_metadata_checked_at":"2026-09-29T04:27:24.220622892Z"},"failed_at":["alert","e2e"],"failed_test_count":9,"occurrences":[{"row_id":"08ea94c0f6dfef4e639c55ac1b6919d0e7a17be44392513f551d05a5119dbc81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","occurred_at":"2026-09-29T01:37:08Z","failed_at":"e2e","signature_id":"85a287d5653ab6bcaf105cd957a8217c394584cc296e9fcdb3b3b96ee45bf756","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0006be1b0\u003e: \n GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c185b878-6f2e-4a60-86e4-2feff70756de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c185b878-6f2e-4a60-86e4-2feff70756de\",\n \"name\": \"c185b878-6f2e-4a60-86e4-2feff70756de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:50:57.370798886Z\",\n \"endTime\": \"2026-09-29T03:20:03.878832155Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0006be1b0\u003e: \n GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c185b878-6f2e-4a60-86e4-2feff70756de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/c185b878-6f2e-4a60-86e4-2feff70756de\",\n \"name\": \"c185b878-6f2e-4a60-86e4-2feff70756de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:50:57.370798886Z\",\n \"endTime\": \"2026-09-29T03:20:03.878832155Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster has no control plane endpoint host; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"11ef539207611e8d9e42037e9088dbd289a275ff0727deaf1fa660ad4b0691a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","occurred_at":"2026-09-29T01:37:08Z","failed_at":"e2e","signature_id":"d438d39959361d9c33251f7a313f500d9482cde722b91ce588c4239bbb94315a","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105c340\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-469757bfrb2w\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e8d80fe2-11a1-436f-9808-5ee06d64084d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e8d80fe2-11a1-436f-9808-5ee06d64084d\",\n \"name\": \"e8d80fe2-11a1-436f-9808-5ee06d64084d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:51:03.349640727Z\",\n \"endTime\": \"2026-09-29T03:20:03.842186988Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 48s ago; version 4.21.35 is Partial (want Completed), started 7m33s ago; version 4.21.35 is Partial (want Completed), started 11m51s ago; version 4.21.35 is Partial (want Completed), started 22m57s ago; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105c340\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-469757bfrb2w\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e8d80fe2-11a1-436f-9808-5ee06d64084d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e8d80fe2-11a1-436f-9808-5ee06d64084d\",\n \"name\": \"e8d80fe2-11a1-436f-9808-5ee06d64084d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:51:03.349640727Z\",\n \"endTime\": \"2026-09-29T03:20:03.842186988Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 48s ago; version 4.21.35 is Partial (want Completed), started 7m33s ago; version 4.21.35 is Partial (want Completed), started 11m51s ago; version 4.21.35 is Partial (want Completed), started 22m57s ago; hosted cluster degraded: UnavailableReplicas: cluster-version-operator deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"95555a98a2afafe44598c5a33e9a57c280d82e0015b35dc42d9e6cb9d9659159","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"1e3ddbeba625177fa9c5ac4d37b95c60c43712181ee859de6c3dda8c47392b22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","occurred_at":"2026-09-29T01:37:08Z","failed_at":"e2e","signature_id":"3b7b441375c2028e92d7a1a20e52f0778ad419ba61d4c3a4875d4e0aa9353139","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-23-pjfds7\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060a5c0\u003e: \n failed to create HCP cluster cluster-zstream-4-23-pjfds7: failed waiting for cluster=\"cluster-zstream-4-23-pjfds7\" in resourcegroup=\"rg-zstream-upgrade-4-23-2ph8ks4m6rn4\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4d243b0-a8bd-4697-acb0-9ef4486ad1e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4d243b0-a8bd-4697-acb0-9ef4486ad1e1\",\n \"name\": \"e4d243b0-a8bd-4697-acb0-9ef4486ad1e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:49:21.332416072Z\",\n \"endTime\": \"2026-09-29T03:18:23.455609493Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-storage-version-migrator, catalog-operator, olm-collect-profiles, redhat-operators-catalog, openshift-route-controller-manager, cluster-policy-controller, cluster-version-operator, hosted-cluster-config-operator, dns-operator, csi-snapshot-controller-operator, control-plane-pki-operator, cluster-storage-operator, cluster-image-registry-operator, kube-scheduler, packageserver, certified-operators-catalog, kube-controller-manager, cluster-node-tuning-operator, cluster-autoscaler, openshift-controller-manager, olm-operator, community-operators-catalog, konnectivity-agent, azure-cloud-controller-manager, cluster-network-operator, machine-approver, openshift-apiserver, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-23-pjfds7\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060a5c0\u003e: \n failed to create HCP cluster cluster-zstream-4-23-pjfds7: failed waiting for cluster=\"cluster-zstream-4-23-pjfds7\" in resourcegroup=\"rg-zstream-upgrade-4-23-2ph8ks4m6rn4\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4d243b0-a8bd-4697-acb0-9ef4486ad1e1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/e4d243b0-a8bd-4697-acb0-9ef4486ad1e1\",\n \"name\": \"e4d243b0-a8bd-4697-acb0-9ef4486ad1e1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:49:21.332416072Z\",\n \"endTime\": \"2026-09-29T03:18:23.455609493Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-storage-version-migrator, catalog-operator, olm-collect-profiles, redhat-operators-catalog, openshift-route-controller-manager, cluster-policy-controller, cluster-version-operator, hosted-cluster-config-operator, dns-operator, csi-snapshot-controller-operator, control-plane-pki-operator, cluster-storage-operator, cluster-image-registry-operator, kube-scheduler, packageserver, certified-operators-catalog, kube-controller-manager, cluster-node-tuning-operator, cluster-autoscaler, openshift-controller-manager, olm-operator, community-operators-catalog, konnectivity-agent, azure-cloud-controller-manager, cluster-network-operator, machine-approver, openshift-apiserver, ingress-operator; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5b287730cc1522c9675d1bc92a45ee5a50996167ec0b230fb0d0298c296c2dc1","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5836ec5561424525964d0ba7f0fa119de61787c2ee2b16c83aec00a1ccde825e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","occurred_at":"2026-09-29T01:37:08Z","failed_at":"e2e","signature_id":"f0d571441c50dded06701c6bc477a7b41c3b63201ffaf54f32862b05ca7933ec","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-lw74fqklhgk4/cluster-candidate-4-21-g2bwxp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f986a0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-g2bwxp: failed waiting for cluster=\"cluster-candidate-4-21-g2bwxp\" in resourcegroup=\"rg-candidate-4-21-lw74fqklhgk4\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5608e9f9-828b-4064-a2f9-7ed4a7817028\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5608e9f9-828b-4064-a2f9-7ed4a7817028\",\n \"name\": \"5608e9f9-828b-4064-a2f9-7ed4a7817028\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:49:24.101243473Z\",\n \"endTime\": \"2026-09-29T03:18:33.772930492Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, catalog-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, olm-operator, cluster-policy-controller, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, openshift-route-controller-manager, ingress-operator, hosted-cluster-config-operator, community-operators-catalog, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-lw74fqklhgk4/cluster-candidate-4-21-g2bwxp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f986a0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-g2bwxp: failed waiting for cluster=\"cluster-candidate-4-21-g2bwxp\" in resourcegroup=\"rg-candidate-4-21-lw74fqklhgk4\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5608e9f9-828b-4064-a2f9-7ed4a7817028\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/5608e9f9-828b-4064-a2f9-7ed4a7817028\",\n \"name\": \"5608e9f9-828b-4064-a2f9-7ed4a7817028\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:49:24.101243473Z\",\n \"endTime\": \"2026-09-29T03:18:33.772930492Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: packageserver, cluster-version-operator, openshift-controller-manager, cluster-node-tuning-operator, catalog-operator, cluster-network-operator, dns-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, olm-operator, cluster-policy-controller, redhat-operators-catalog, redhat-marketplace-catalog, certified-operators-catalog, openshift-route-controller-manager, ingress-operator, hosted-cluster-config-operator, community-operators-catalog, cluster-storage-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 2 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28485ae47900ca737aa0bfad6d457ebe9972f9b9dbd85391d03a17bd274e9f55","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"596e006be4ef5dec95e6a874144b86f8ca5afa73b2e25fa5164ab11783aea109","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","occurred_at":"2026-09-29T01:37:08Z","failed_at":"e2e","signature_id":"977e88260a2c63839e01a7379ef3eb4856b48c29fe506cf53d597be69eae5a25","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-gsz2fzdqj9cx/cluster-candidate-4-23-t8dbk2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012024a0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-t8dbk2: failed waiting for cluster=\"cluster-candidate-4-23-t8dbk2\" in resourcegroup=\"rg-candidate-4-23-gsz2fzdqj9cx\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f996b9-ba98-44c4-95fb-3d754f91c4bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f996b9-ba98-44c4-95fb-3d754f91c4bd\",\n \"name\": \"f4f996b9-ba98-44c4-95fb-3d754f91c4bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:56:27.709492276Z\",\n \"endTime\": \"2026-09-29T03:25:33.644090026Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 24m4s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-gsz2fzdqj9cx/cluster-candidate-4-23-t8dbk2 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012024a0\u003e: \n failed to create HCP cluster cluster-candidate-4-23-t8dbk2: failed waiting for cluster=\"cluster-candidate-4-23-t8dbk2\" in resourcegroup=\"rg-candidate-4-23-gsz2fzdqj9cx\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f996b9-ba98-44c4-95fb-3d754f91c4bd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/f4f996b9-ba98-44c4-95fb-3d754f91c4bd\",\n \"name\": \"f4f996b9-ba98-44c4-95fb-3d754f91c4bd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:56:27.709492276Z\",\n \"endTime\": \"2026-09-29T03:25:33.644090026Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.23.0-ec.1 is Partial (want Completed), started 24m4s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"61eb2c12b09e23e814a33fdf3b54b2ca24cabadce2393eb8c7b32403a25c2a08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","occurred_at":"2026-09-29T01:37:08Z","failed_at":"alert","signature_id":"74ab7a38a487dbdf27bb936e43656706a326d41cf93afeca61974d10b41ebea3","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s) (1 unknown, 7 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T02:59:20Z\n Ended: 2026-09-29T03:22:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5787776-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t4dichoa89988qtbcmigs9shnp9rnj0-f8k7m4c1j4r1t2k\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t4dichoa89988qtbcmigs9shnp9rnj0-f8k7m4c1j4r1t2k/etcd-1 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"6fd6612547f60c0c78526b6473d89d892f89449dcd34a9b6ff54e5008402fa9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","occurred_at":"2026-09-29T01:37:08Z","failed_at":"e2e","signature_id":"325116679ad1eeacef1b16c81bf441b9fdf1e7305cde72865d3ac779204218de","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bfa500\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-s6psdk49m28r\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edddc05b-21c8-4d1e-832c-40b942f545e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edddc05b-21c8-4d1e-832c-40b942f545e9\",\n \"name\": \"edddc05b-21c8-4d1e-832c-40b942f545e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:50:21.626128144Z\",\n \"endTime\": \"2026-09-29T03:19:23.597463388Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, cluster-version-operator, openshift-route-controller-manager, cluster-image-registry-operator, openshift-controller-manager, cluster-storage-operator, olm-operator, certified-operators-catalog, dns-operator, cluster-policy-controller, catalog-operator, community-operators-catalog, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bfa500\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-s6psdk49m28r\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edddc05b-21c8-4d1e-832c-40b942f545e9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edddc05b-21c8-4d1e-832c-40b942f545e9\",\n \"name\": \"edddc05b-21c8-4d1e-832c-40b942f545e9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:50:21.626128144Z\",\n \"endTime\": \"2026-09-29T03:19:23.597463388Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: cluster-node-tuning-operator, ingress-operator, redhat-operators-catalog, redhat-marketplace-catalog, hosted-cluster-config-operator, csi-snapshot-controller-operator, packageserver, cluster-version-operator, openshift-route-controller-manager, cluster-image-registry-operator, openshift-controller-manager, cluster-storage-operator, olm-operator, certified-operators-catalog, dns-operator, cluster-policy-controller, catalog-operator, community-operators-catalog, cluster-network-operator; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, certified-operators-catalog deployment has 1 unavailable replicas, cluster-network-operator deployment has 1 unavailable replicas, cluster-storage-operator deployment has 1 unavailable replicas, cluster-version-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, csi-snapshot-controller-operator deployment has 1 unavailable replicas, dns-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, ingress-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28485ae47900ca737aa0bfad6d457ebe9972f9b9dbd85391d03a17bd274e9f55","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: multiple components; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"786ec1e5957d0a17c69200884e665efdd05d8aaa54b442594c9354f644b34e9e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","occurred_at":"2026-09-29T01:37:08Z","failed_at":"e2e","signature_id":"7df035b98ec6828f15f876568df12cc88b8f42de389305b1acc7b6de00986d18","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c18980\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-vspgr4ltfkks\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/81a89006-e9e6-43e0-a945-6223e2371587\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/81a89006-e9e6-43e0-a945-6223e2371587\",\n \"name\": \"81a89006-e9e6-43e0-a945-6223e2371587\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:49:48.446308819Z\",\n \"endTime\": \"2026-09-29T03:18:53.683107891Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, ingress-operator, cluster-image-registry-operator, cluster-version-operator, hosted-cluster-config-operator, certified-operators-catalog, cluster-storage-operator, cluster-node-tuning-operator, cluster-policy-controller, catalog-operator, packageserver, redhat-operators-catalog, csi-snapshot-controller-operator, olm-operator, openshift-controller-manager, cluster-network-operator, dns-operator, community-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c18980\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-vspgr4ltfkks\" to finish creating: GET https://rp.j5787776.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/81a89006-e9e6-43e0-a945-6223e2371587\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/81a89006-e9e6-43e0-a945-6223e2371587\",\n \"name\": \"81a89006-e9e6-43e0-a945-6223e2371587\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:49:48.446308819Z\",\n \"endTime\": \"2026-09-29T03:18:53.683107891Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: openshift-route-controller-manager, ingress-operator, cluster-image-registry-operator, cluster-version-operator, hosted-cluster-config-operator, certified-operators-catalog, cluster-storage-operator, cluster-node-tuning-operator, cluster-policy-controller, catalog-operator, packageserver, redhat-operators-catalog, csi-snapshot-controller-operator, olm-operator, openshift-controller-manager, cluster-network-operator, dns-operator, community-operators-catalog, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2c0a88f2a0ef84e4c12e30b05c3779aee6f0cf66551b88fac3c1693eccdbecc5","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, olm-operator, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f54174457e44bf8eaddbc4a4e5d2e92f003c4b37f0e3c6f76d9c2294a49c8353","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104747271315787776","occurred_at":"2026-09-29T01:37:08Z","failed_at":"alert","signature_id":"f61af137b1b5026eee00786cad11ec344757635c9f98c88913206edf037c918d","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T03:16:20Z\n Ended: 2026-09-29T03:23:21Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j5787776-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.87:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5787776/providers/microsoft.monitor/accounts/hcps-j5787776\", namespace=\"ocm-arohcpci01-2t4dichoa89988qtbcmigs9shnp9rnj0-f8k7m4c1j4r1t2k\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-7q7bt\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t4dichoa89988qtbcmigs9shnp9rnj0-f8k7m4c1j4r1t2k/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["28485ae47900ca737aa0bfad6d457ebe9972f9b9dbd85391d03a17bd274e9f55","2c0a88f2a0ef84e4c12e30b05c3779aee6f0cf66551b88fac3c1693eccdbecc5","5b287730cc1522c9675d1bc92a45ee5a50996167ec0b230fb0d0298c296c2dc1","7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","8ba35a0be8ccf0b303098704ed0163e70d6e62fb03a868282a3dd05f8e223ca9","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","95555a98a2afafe44598c5a33e9a57c280d82e0015b35dc42d9e6cb9d9659159","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104737117333622784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"3e077fa473e1d0fb9f27fc5472f089cc2bc994d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-29T00:56:46Z","started_at":"2026-09-29T00:56:46Z","completed_at":"2026-09-29T03:25:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T03:35:50.420899003Z","timing_metadata_checked_at":"2026-09-29T03:35:50.420899003Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T03:35:50.425621688Z","region_metadata_checked_at":"2026-09-29T03:35:50.425621688Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7168/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104727513765777408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7168,"pr_state":"closed","pr_sha":"438a454514812b6dc5eb79970e7dce00ce113844","final_merged_sha":"438a454514812b6dc5eb79970e7dce00ce113844","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T00:18:37Z","started_at":"2026-09-29T00:18:37Z","completed_at":"2026-09-29T02:36:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T02:42:58.20445325Z","timing_metadata_checked_at":"2026-09-29T02:42:58.20445325Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T02:42:58.209019835Z","region_metadata_checked_at":"2026-09-29T02:42:58.209019835Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"int","summary":{"total_runs":6,"failed_runs":4,"runs_with_occurrences":4,"runs_with_failure_pattern_matches":4,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105040832624070656","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T21:03:38Z","started_at":"2026-09-29T21:03:38Z","completed_at":"2026-09-29T23:41:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T23:51:07.306596512Z","timing_metadata_checked_at":"2026-09-29T23:51:07.306596512Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T23:51:07.309745102Z","region_metadata_checked_at":"2026-09-29T23:51:07.309745102Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"85bd1f835147fc2021f054a8bf27e3fefe6b32579972fe7fcd9d6c70909fae4d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105040832624070656","occurred_at":"2026-09-29T21:03:38Z","failed_at":"e2e","signature_id":"3b25e49ebcf01ceefe5ef6d15772a6b3207794a1d640c359ac345895674ed15c","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00134a240\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lf58x6mrt8sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\",\n \"name\": \"6899b842-6be7-4752-9378-45ead2f46f38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T22:48:49.695468888Z\",\n \"endTime\": \"2026-09-29T23:07:57.514241775Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00134a240\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-lf58x6mrt8sn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6899b842-6be7-4752-9378-45ead2f46f38\",\n \"name\": \"6899b842-6be7-4752-9378-45ead2f46f38\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T22:48:49.695468888Z\",\n \"endTime\": \"2026-09-29T23:07:57.514241775Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-lf58x6mrt8sn--managed/providers/Microsoft.Compute/virtualMachines/j1d5g5c7r6t3f3c-autoscale-noaz-hllff-7t674 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e5568b11d6d0902b98c685a3c0510e57a63a12ce81a618b3ddbec2eedc7730b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2105040832624070656","occurred_at":"2026-09-29T21:03:38Z","failed_at":"e2e","signature_id":"607e94a35a81473d478091f2da267ba95cac66f639d687416633136e0271f1e0","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010311c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-zbbkqjx7fwrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\",\n \"name\": \"15a80b6b-fe14-4fa6-a826-e605868e189f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T21:44:03.296557Z\",\n \"endTime\": \"2026-09-29T22:03:11.679917905Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010311c0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-zbbkqjx7fwrv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a80b6b-fe14-4fa6-a826-e605868e189f\",\n \"name\": \"15a80b6b-fe14-4fa6-a826-e605868e189f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T21:44:03.296557Z\",\n \"endTime\": \"2026-09-29T22:03:11.679917905Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-kclwz: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-kclwz to exist\\nMachine e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-zbbkqjx7fwrv--managed/providers/Microsoft.Compute/virtualMachines/e3b7f3p1b7r7k1s-np-1-djfk4-t2vnm to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104966711995273216","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T16:09:07Z","started_at":"2026-09-29T16:09:07Z","completed_at":"2026-09-29T18:35:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T18:36:50.996686342Z","timing_metadata_checked_at":"2026-09-29T18:36:50.996686342Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T18:36:51.002413324Z","region_metadata_checked_at":"2026-09-29T18:36:51.002413324Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"701e26cd2f9735fcabd7e2e1780b1057d200ccd5ae566683c41e046efd1df191","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104966711995273216","occurred_at":"2026-09-29T16:09:07Z","failed_at":"e2e","signature_id":"464235197a1662fe879ebc18d73f5450d313eafaf40b203a076ea03d5c5ec2ee","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f32080\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-thvwrhnx9f2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n \"name\": \"aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T16:37:13.215287875Z\",\n \"endTime\": \"2026-09-29T16:56:16.225730373Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f32080\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-thvwrhnx9f2p\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n \"name\": \"aa62eeea-3561-4f5f-872a-af727c8e3ec4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T16:37:13.215287875Z\",\n \"endTime\": \"2026-09-29T16:56:16.225730373Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-jmfl5 to exist\\nMachine e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-thvwrhnx9f2p--managed/providers/Microsoft.Compute/virtualMachines/e2v2e3f8a5f0f0j-np-1-pksnh-vpvtc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d94d41abb3b7734e8c90a8fbe43ef5a58bb24dd70087bb3075a473a7b7242d3f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104966711995273216","occurred_at":"2026-09-29T16:09:07Z","failed_at":"e2e","signature_id":"3f1b5d3d9c68310f2e07573951e1f580c91e75043ece8b8cef2bcc6b19f1f6b8","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00143c978\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00143c978\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104927790917750784","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T13:34:27Z","started_at":"2026-09-29T13:34:27Z","completed_at":"2026-09-29T16:03:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T16:10:32.61749758Z","timing_metadata_checked_at":"2026-09-29T16:10:32.61749758Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T16:10:32.622228764Z","region_metadata_checked_at":"2026-09-29T16:10:32.622228764Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"45a34f13f41fded10d774e60d54af8c83ac821652e84e79576fe99ee8ad005b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104927790917750784","occurred_at":"2026-09-29T13:34:27Z","failed_at":"e2e","signature_id":"90bf9bad3db34ef7342fcb200f948ba1cd05e545d40dcef95246cf50da7d0cc4","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00090a4c8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.006s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00090a4c8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"c35162e5bd75a32a4e6a54be18f2624b1b28d4d7e0251835d9388582d1b6b1a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104927790917750784","occurred_at":"2026-09-29T13:34:27Z","failed_at":"e2e","signature_id":"b47ba10dff74b3c276896ac768dedd60fa50be8640a5fc7175dfae6a540b5f0c","test_name":"Customer should create an HCP cluster and validate TLS certificates","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009bd5e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-2njpnbrz5jg5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e034572f-71f9-4484-a06b-08d411aba3d9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e034572f-71f9-4484-a06b-08d411aba3d9\",\n \"name\": \"e034572f-71f9-4484-a06b-08d411aba3d9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T14:28:57.448756437Z\",\n \"endTime\": \"2026-09-29T14:48:03.628370174Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n1q8y3e1s2j2m9q-np-1-spldz-2nngd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2njpnbrz5jg5--managed/providers/Microsoft.Compute/virtualMachines/n1q8y3e1s2j2m9q-np-1-spldz-2nngd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n1q8y3e1s2j2m9q-np-1-spldz-2nngd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2njpnbrz5jg5--managed/providers/Microsoft.Compute/virtualMachines/n1q8y3e1s2j2m9q-np-1-spldz-2nngd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_tls_endpoints.go:156]: failed to create node pool np-1 for TLS endpoint cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009bd5e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"tls-endpoint-hcp-cluster\" in resourcegroup=\"tls-endpoint-cluster-2njpnbrz5jg5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e034572f-71f9-4484-a06b-08d411aba3d9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e034572f-71f9-4484-a06b-08d411aba3d9\",\n \"name\": \"e034572f-71f9-4484-a06b-08d411aba3d9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T14:28:57.448756437Z\",\n \"endTime\": \"2026-09-29T14:48:03.628370174Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n1q8y3e1s2j2m9q-np-1-spldz-2nngd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2njpnbrz5jg5--managed/providers/Microsoft.Compute/virtualMachines/n1q8y3e1s2j2m9q-np-1-spldz-2nngd to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n1q8y3e1s2j2m9q-np-1-spldz-2nngd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/tls-endpoint-cluster-2njpnbrz5jg5--managed/providers/Microsoft.Compute/virtualMachines/n1q8y3e1s2j2m9q-np-1-spldz-2nngd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104877329665757184","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T10:13:56Z","started_at":"2026-09-29T10:13:56Z","completed_at":"2026-09-29T12:36:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T12:50:34.590538421Z","timing_metadata_checked_at":"2026-09-29T12:50:34.590538421Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T12:50:34.596649409Z","region_metadata_checked_at":"2026-09-29T12:50:34.596649409Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dd86161a5ecafec54a4670bc3ba3e5a4be04874ccba82a952038f7aacc338e14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104877329665757184","occurred_at":"2026-09-29T10:13:56Z","failed_at":"e2e","signature_id":"cb0b865ce23da66a9ce61d2cfe51d7b9448f029a36bab5dde0e72f51033a5d6f","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00061eb40\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.000s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00061eb40\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["6ae8052fe24068dd122c1d2c69a1fea3114083e0220320db69d2f9ee76d7c61d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104823044923985920","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T06:38:14Z","started_at":"2026-09-29T06:38:14Z","completed_at":"2026-09-29T08:50:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:00:37.097647876Z","timing_metadata_checked_at":"2026-09-29T09:00:37.097647876Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:00:37.100141268Z","region_metadata_checked_at":"2026-09-29T09:00:37.100141268Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104766693522804736","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T02:54:19Z","started_at":"2026-09-29T02:54:19Z","completed_at":"2026-09-29T05:14:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T05:29:31.113800237Z","timing_metadata_checked_at":"2026-09-29T05:29:31.113800237Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T05:29:31.117621725Z","region_metadata_checked_at":"2026-09-29T05:29:31.117621725Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":7,"failed_runs":6,"runs_with_occurrences":6,"runs_with_failure_pattern_matches":6,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T23:00:27Z","started_at":"2026-09-29T23:00:27Z","completed_at":"2026-09-30T00:50:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T01:04:32.96599271Z","timing_metadata_checked_at":"2026-09-30T01:04:32.96599271Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T01:04:32.969954798Z","region_metadata_checked_at":"2026-09-30T01:04:32.969954798Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"01d4c45bfb55769aad1a108d71ef3fcd81216065478151431c7fafd717499667","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"ab81972b124c700f296b8a673e81e3c45e989c5f882990bc542bbc011d3afe4b","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0025a1e90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g9r75fj7b84l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0025a1e90\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-g9r75fj7b84l/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"0647fd8b6ca18e12e6049a83b3d5bb59e447251045334f9d5557a3ce9821e965","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"b5053c3b2591063eebcca7305d3b183c2d49e8bb593fee7a912391475b426d8b","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ec1320\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-cbk98wj7cx2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a41762b8-220b-4e53-b3be-168dfb531e91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a41762b8-220b-4e53-b3be-168dfb531e91\",\n \"name\": \"a41762b8-220b-4e53-b3be-168dfb531e91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:10:58.443702839Z\",\n \"endTime\": \"2026-09-29T23:30:01.681918109Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k2r6f6w3v3g4c8f-np-1-psrbp-v6w6d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-cbk98wj7cx2h--managed/providers/Microsoft.Compute/virtualMachines/k2r6f6w3v3g4c8f-np-1-psrbp-v6w6d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k2r6f6w3v3g4c8f-np-1-psrbp-v6w6d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-cbk98wj7cx2h--managed/providers/Microsoft.Compute/virtualMachines/k2r6f6w3v3g4c8f-np-1-psrbp-v6w6d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ec1320\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-cbk98wj7cx2h\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a41762b8-220b-4e53-b3be-168dfb531e91\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a41762b8-220b-4e53-b3be-168dfb531e91\",\n \"name\": \"a41762b8-220b-4e53-b3be-168dfb531e91\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:10:58.443702839Z\",\n \"endTime\": \"2026-09-29T23:30:01.681918109Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine k2r6f6w3v3g4c8f-np-1-psrbp-v6w6d: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-cbk98wj7cx2h--managed/providers/Microsoft.Compute/virtualMachines/k2r6f6w3v3g4c8f-np-1-psrbp-v6w6d to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine k2r6f6w3v3g4c8f-np-1-psrbp-v6w6d: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-cbk98wj7cx2h--managed/providers/Microsoft.Compute/virtualMachines/k2r6f6w3v3g4c8f-np-1-psrbp-v6w6d to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"0ae6c70b97f362388f4380120d584828209ccec89e4150b4d816f7ebb2cb340a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"b0d7ed30747f61a6a3d756941cf566b67b31d3433ed9d4a53f42e2ac0a512800","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0016fb248\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-29-110018, targetMinor=5.1.0-0.nightly-multi-2026-09-29-141926) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-29-141926\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0016fb248\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-29-110018, targetMinor=5.1.0-0.nightly-multi-2026-09-29-141926) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-29-141926\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"20358267ef386f35952a78bab804ee9b5e869644516cc234932b69c4bc6da528","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"984c5418f5dedc5b0ed598f0905dc01fee59d41405a1baeafdc73a20550d35c4","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-76zbc6\" for cluster \"cluster-nightly-4-20-76zbc6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010d40e0\u003e: \n failed to create NodePool np-76zbc6: failed starting nodepool creation \"np-76zbc6\" for cluster \"cluster-nightly-4-20-76zbc6\" in resourcegroup=\"rg-nightly-4-20-tmbd62wjjmv5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-tmbd62wjjmv5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-76zbc6/nodePools/np-76zbc6\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-76zbc6\" for cluster \"cluster-nightly-4-20-76zbc6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010d40e0\u003e: \n failed to create NodePool np-76zbc6: failed starting nodepool creation \"np-76zbc6\" for cluster \"cluster-nightly-4-20-76zbc6\" in resourcegroup=\"rg-nightly-4-20-tmbd62wjjmv5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-tmbd62wjjmv5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-76zbc6/nodePools/np-76zbc6\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"22ab30cc35d4717831138c83092226338f46c930162c6814d801ad460869f023","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"8d39fe7701ab40dc16efae981b58fc0b69e6618000408f613b10fbc896344f79","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012dbc40\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-hz8wft\" resourcegroup=\"rg-np-dg-minor-hz8wft-2td2rb8jcw6z\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-hz8wft-2td2rb8jcw6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-hz8wft/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012dbc40\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-hz8wft\" resourcegroup=\"rg-np-dg-minor-hz8wft-2td2rb8jcw6z\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-hz8wft-2td2rb8jcw6z/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-hz8wft/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"34653ff4d97a35e22b29ea2c7d3da2523289140c8b9a2a5afdeafb4ea266a11f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"50207d433bd8fc8889a086afbcfe6cc2a83818780d07860dbd746ab6aa5f696a","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002153e30\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-ndd4ps9rz69q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002153e30\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-ndd4ps9rz69q/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"69bc23e88ba22670d42c31d1a91cd53136da216f5482f212dba64ed9327a1098","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"a3e4caf92ff026405a7e1f6c2094f6a5dd4d785e788eebaec8e32b6d50d14170","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d87400\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cg4srqjqkk9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/49815947-d474-46af-9a2a-1b7e7d2030b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/49815947-d474-46af-9a2a-1b7e7d2030b2\",\n \"name\": \"49815947-d474-46af-9a2a-1b7e7d2030b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:56:01.394801153Z\",\n \"endTime\": \"2026-09-30T00:15:03.380581382Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l3i7y0e6s8b1j1r-np-delete-main-5wq22-8rjb7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cg4srqjqkk9n--managed/providers/Microsoft.Compute/virtualMachines/l3i7y0e6s8b1j1r-np-delete-main-5wq22-8rjb7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l3i7y0e6s8b1j1r-np-delete-main-5wq22-8rjb7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cg4srqjqkk9n--managed/providers/Microsoft.Compute/virtualMachines/l3i7y0e6s8b1j1r-np-delete-main-5wq22-8rjb7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000d87400\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-cg4srqjqkk9n\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/49815947-d474-46af-9a2a-1b7e7d2030b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/49815947-d474-46af-9a2a-1b7e7d2030b2\",\n \"name\": \"49815947-d474-46af-9a2a-1b7e7d2030b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T23:56:01.394801153Z\",\n \"endTime\": \"2026-09-30T00:15:03.380581382Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l3i7y0e6s8b1j1r-np-delete-main-5wq22-8rjb7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cg4srqjqkk9n--managed/providers/Microsoft.Compute/virtualMachines/l3i7y0e6s8b1j1r-np-delete-main-5wq22-8rjb7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l3i7y0e6s8b1j1r-np-delete-main-5wq22-8rjb7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-cg4srqjqkk9n--managed/providers/Microsoft.Compute/virtualMachines/l3i7y0e6s8b1j1r-np-delete-main-5wq22-8rjb7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lwltp7hhv9xn\" to finish creating: GET https://management"}},{"row_id":"6adf2b3595a5c8bf5d37f2c66d6840a72e4482d419bd0848af0cd6bf79495b26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"64d606c7013b8cfeb404ee0170215261a668cce8e28af13e69b08aea931ad7ae","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-nightly-4-21-lhp5d5\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0xc001347770\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:135]: failed to verify HCP cluster \"cluster-nightly-4-21-lhp5d5\" is viable\nUnexpected error:\n \u003c*errors.joinError | 0xc001347770\u003e: \n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Unauthorized\n VerifyBasicAccess failed: failed to list services: Unauthorized\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967","failure_pattern":"VerifyBasicAccess failed: failed to list services: Unauthorized","search_query":"VerifyBasicAccess failed: failed to list services: Unauthorized"}},{"row_id":"b31f217010c7919e947c723eb804f035c28eb9ed33d1d45482c9f741a7f1f275","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"7818745f304feb896fd9956f5ee3670f59a498424470ff28f7586ab443157495","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000800580\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-v4bb8q\" in resourcegroup=\"rg-np-version-upgrade-v4bb8q-6wwsb7bm4cjc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-v4bb8q-6wwsb7bm4cjc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-v4bb8q/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000800580\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-v4bb8q\" in resourcegroup=\"rg-np-version-upgrade-v4bb8q-6wwsb7bm4cjc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-v4bb8q-6wwsb7bm4cjc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-v4bb8q/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b3cfc0244c2c266fe7d6315b8541348a342db0f112d68beeb60d93698178f5c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"ac2796a0eaea7e746033ae3b5d11b56ebdc0bb718667ec8bb359df5053e65b9e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0029fc4b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bgcgl9s2h6qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0029fc4b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-bgcgl9s2h6qx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b3dbfe4a176b423f1ab4d921d879495bfd4081d4bcbed7631c086dd7e873f61d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"0f5a77fa6e7fa38d8419561ee65cf359756f5ea87b9f69a8ab747b2213725721","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb6040\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-dzh29w\" in resourcegroup=\"rg-np-skip-minor-dzh29w-lnc5s2hc2m99\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-dzh29w-lnc5s2hc2m99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-dzh29w/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb6040\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-dzh29w\" in resourcegroup=\"rg-np-skip-minor-dzh29w-lnc5s2hc2m99\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-dzh29w-lnc5s2hc2m99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-dzh29w/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d44afae896c5eae708743da5cae070f4ac2b200d935d546d64dd282fd86aa00e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"5fe9c90af475229134a22e64f783a28fa23b2ed7adb7eeb62eb4172c9077d547","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015ed110\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-scbxjjvdnhlm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0015ed110\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-scbxjjvdnhlm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"d58c0ec236209501e56130d841fae2f33145018e33eef5d4ce63fe3d3761005a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2105070229963935744","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"eb590a9e4b21d9f9344043e4b421a3036217950639822dca5e155bb23a7e960a","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000fcf3e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-mnsnt8965m6w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000fcf3e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-mnsnt8965m6w/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","fee9bdd5d83c8738b8bf37e9ed6800aab39015cbc2ff9fc672a130098b9fd967"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104965516199202816","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T16:04:17Z","started_at":"2026-09-29T16:04:17Z","completed_at":"2026-09-29T18:35:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T18:47:29.251338127Z","timing_metadata_checked_at":"2026-09-29T18:47:29.251338127Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T18:47:29.254831716Z","region_metadata_checked_at":"2026-09-29T18:47:29.254831716Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0e8c25f9491e2ccb0936470cfffd1e54ce20141e11cd82b77f0ca237da6ffe2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104965516199202816","occurred_at":"2026-09-29T16:04:17Z","failed_at":"e2e","signature_id":"29c6308b61ce9ac98fc2b6f75fa558b79a87d544d6d16932cf4ac319fcc54ddd","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:740]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022120\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-kvm5ll\" in resourcegroup=\"rg-np-dg-minor-kvm5ll-6sf2vr9vw4v6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42494991-e3af-48e6-9cf3-a4fc4d279c44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42494991-e3af-48e6-9cf3-a4fc4d279c44\",\n \"name\": \"42494991-e3af-48e6-9cf3-a4fc4d279c44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:48:38.482509384Z\",\n \"endTime\": \"2026-09-29T18:07:42.157683685Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z0x4w0e5g3t9x5d-npdg-422-420-slkn4-nbjwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-kvm5ll-6sf2vr9vw4v6-np-dgm-kvm5ll--managed/providers/Microsoft.Compute/virtualMachines/z0x4w0e5g3t9x5d-npdg-422-420-slkn4-nbjwr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z0x4w0e5g3t9x5d-npdg-422-420-slkn4-nbjwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-kvm5ll-6sf2vr9vw4v6-np-dgm-kvm5ll--managed/providers/Microsoft.Compute/virtualMachines/z0x4w0e5g3t9x5d-npdg-422-420-slkn4-nbjwr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:740]: failed to create nodepool npdg-422-420\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001022120\u003e: \n failed to create NodePool npdg-422-420: failed waiting for nodepool=\"npdg-422-420\" for cluster \"np-dg-minor-kvm5ll\" in resourcegroup=\"rg-np-dg-minor-kvm5ll-6sf2vr9vw4v6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42494991-e3af-48e6-9cf3-a4fc4d279c44\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/42494991-e3af-48e6-9cf3-a4fc4d279c44\",\n \"name\": \"42494991-e3af-48e6-9cf3-a4fc4d279c44\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:48:38.482509384Z\",\n \"endTime\": \"2026-09-29T18:07:42.157683685Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z0x4w0e5g3t9x5d-npdg-422-420-slkn4-nbjwr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-kvm5ll-6sf2vr9vw4v6-np-dgm-kvm5ll--managed/providers/Microsoft.Compute/virtualMachines/z0x4w0e5g3t9x5d-npdg-422-420-slkn4-nbjwr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z0x4w0e5g3t9x5d-npdg-422-420-slkn4-nbjwr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-kvm5ll-6sf2vr9vw4v6-np-dgm-kvm5ll--managed/providers/Microsoft.Compute/virtualMachines/z0x4w0e5g3t9x5d-npdg-422-420-slkn4-nbjwr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4b30c014beec4ba4e7151e14dc7fe90ec14fb95aa02364536a926435846dae02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104965516199202816","occurred_at":"2026-09-29T16:04:17Z","failed_at":"e2e","signature_id":"71baf30e358dcc92a29c095b60aa9044f354e28a4227a5d23e392170a482f649","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3a060\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-fld425vx7n2r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/84d85252-8c39-447d-8d5d-a497997a37f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/84d85252-8c39-447d-8d5d-a497997a37f5\",\n \"name\": \"84d85252-8c39-447d-8d5d-a497997a37f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T16:34:49.001481538Z\",\n \"endTime\": \"2026-09-29T16:53:58.101757577Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-l7x62-h42h7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-fld425vx7n2r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-l7x62-h42h7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-l7x62-h42h7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-fld425vx7n2r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-l7x62-h42h7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3a060\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-fld425vx7n2r\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/84d85252-8c39-447d-8d5d-a497997a37f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/84d85252-8c39-447d-8d5d-a497997a37f5\",\n \"name\": \"84d85252-8c39-447d-8d5d-a497997a37f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T16:34:49.001481538Z\",\n \"endTime\": \"2026-09-29T16:53:58.101757577Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine private-ingress-np-1-l7x62-h42h7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-fld425vx7n2r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-l7x62-h42h7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine private-ingress-np-1-l7x62-h42h7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-fld425vx7n2r--managed/providers/Microsoft.Compute/virtualMachines/private-ingress-np-1-l7x62-h42h7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a573aa349f22080b2666811aa4c57f2b7df935cce5449d502512b03f3fb3c55c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104965516199202816","occurred_at":"2026-09-29T16:04:17Z","failed_at":"e2e","signature_id":"43c2d0116824b83e8b858f491f99a45e89dc95981d17cfa15a10257c4534a1e4","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00079ab80\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lwltp7hhv9xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ade802f3-fc4a-4348-8b7b-a05cecc41cb9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ade802f3-fc4a-4348-8b7b-a05cecc41cb9\",\n \"name\": \"ade802f3-fc4a-4348-8b7b-a05cecc41cb9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:40:59.55523927Z\",\n \"endTime\": \"2026-09-29T18:00:02.471778007Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w8d5r4n4c6l8f1n-np-deletable-5mg5s-5v87h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-lwltp7hhv9xn--managed/providers/Microsoft.Compute/virtualMachines/w8d5r4n4c6l8f1n-np-deletable-5mg5s-5v87h to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w8d5r4n4c6l8f1n-np-deletable-5mg5s-5v87h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-lwltp7hhv9xn--managed/providers/Microsoft.Compute/virtualMachines/w8d5r4n4c6l8f1n-np-deletable-5mg5s-5v87h to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc00079ab80\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lwltp7hhv9xn\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ade802f3-fc4a-4348-8b7b-a05cecc41cb9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ade802f3-fc4a-4348-8b7b-a05cecc41cb9\",\n \"name\": \"ade802f3-fc4a-4348-8b7b-a05cecc41cb9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T17:40:59.55523927Z\",\n \"endTime\": \"2026-09-29T18:00:02.471778007Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w8d5r4n4c6l8f1n-np-deletable-5mg5s-5v87h: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-lwltp7hhv9xn--managed/providers/Microsoft.Compute/virtualMachines/w8d5r4n4c6l8f1n-np-deletable-5mg5s-5v87h to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w8d5r4n4c6l8f1n-np-deletable-5mg5s-5v87h: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-lwltp7hhv9xn--managed/providers/Microsoft.Compute/virtualMachines/w8d5r4n4c6l8f1n-np-deletable-5mg5s-5v87h to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lwltp7hhv9xn\" to finish creating: GET https://management"}},{"row_id":"f35de9639b067facf2ec43d8fb0b933149cbb1705ecdce00eebbe0801f5bcfbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104965516199202816","occurred_at":"2026-09-29T16:04:17Z","failed_at":"e2e","signature_id":"707a57c38a5f874a221b05580419af4288b711c6c3d542a38d17de6bbd9c1baa","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000deacc0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.001s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000deacc0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104910007010070528","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T12:23:47Z","started_at":"2026-09-29T12:23:47Z","completed_at":"2026-09-29T15:00:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T15:07:30.631649041Z","timing_metadata_checked_at":"2026-09-29T15:07:30.631649041Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T15:07:30.634503131Z","region_metadata_checked_at":"2026-09-29T15:07:30.634503131Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"000e3952784a563959113896f154c5e9a4409bfd78b2250b9b78dad12cf8add0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104910007010070528","occurred_at":"2026-09-29T12:23:47Z","failed_at":"e2e","signature_id":"ff3cec3db7295b1ce4aa8575944947a6dd1653b69e1732f1307befed4fa560d2","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f52a00\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-42jhzb\" in resourcegroup=\"rg-np-noedge-42jhzb-sgqdpkmgvtp6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b11a0a62-0163-43ee-bbd8-7683a7459b17\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b11a0a62-0163-43ee-bbd8-7683a7459b17\",\n \"name\": \"b11a0a62-0163-43ee-bbd8-7683a7459b17\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T14:07:28.714225296Z\",\n \"endTime\": \"2026-09-29T14:26:31.083712333Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z6e0a9g3g7e7q4w-npnoedge-4-20-jjt24-gstcv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-42jhzb-sgqdpkmgvtp6-np-ne-42jhzb--managed/providers/Microsoft.Compute/virtualMachines/z6e0a9g3g7e7q4w-npnoedge-4-20-jjt24-gstcv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z6e0a9g3g7e7q4w-npnoedge-4-20-jjt24-gstcv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-42jhzb-sgqdpkmgvtp6-np-ne-42jhzb--managed/providers/Microsoft.Compute/virtualMachines/z6e0a9g3g7e7q4w-npnoedge-4-20-jjt24-gstcv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f52a00\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-42jhzb\" in resourcegroup=\"rg-np-noedge-42jhzb-sgqdpkmgvtp6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b11a0a62-0163-43ee-bbd8-7683a7459b17\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/b11a0a62-0163-43ee-bbd8-7683a7459b17\",\n \"name\": \"b11a0a62-0163-43ee-bbd8-7683a7459b17\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T14:07:28.714225296Z\",\n \"endTime\": \"2026-09-29T14:26:31.083712333Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine z6e0a9g3g7e7q4w-npnoedge-4-20-jjt24-gstcv: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-42jhzb-sgqdpkmgvtp6-np-ne-42jhzb--managed/providers/Microsoft.Compute/virtualMachines/z6e0a9g3g7e7q4w-npnoedge-4-20-jjt24-gstcv to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine z6e0a9g3g7e7q4w-npnoedge-4-20-jjt24-gstcv: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-42jhzb-sgqdpkmgvtp6-np-ne-42jhzb--managed/providers/Microsoft.Compute/virtualMachines/z6e0a9g3g7e7q4w-npnoedge-4-20-jjt24-gstcv to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"3299c06a22e870b1781388dc1afaa9dd93cc17c4d2e2912b87acee1cba85dc10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104910007010070528","occurred_at":"2026-09-29T12:23:47Z","failed_at":"e2e","signature_id":"9073621679b1c76fd360d5a0f2ff74c48942581ff1cc5010cbd7c39daf18c3d9","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000dfcbc0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r2grpqzmpxnk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2b31ee5-5378-49f4-b4f9-8c82f9bace93\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2b31ee5-5378-49f4-b4f9-8c82f9bace93\",\n \"name\": \"e2b31ee5-5378-49f4-b4f9-8c82f9bace93\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T13:59:14.617140907Z\",\n \"endTime\": \"2026-09-29T14:18:21.033986186Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r0n8s9h5n0s8u3c-np-delete-main-5vq47-vdkbs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r2grpqzmpxnk--managed/providers/Microsoft.Compute/virtualMachines/r0n8s9h5n0s8u3c-np-delete-main-5vq47-vdkbs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r0n8s9h5n0s8u3c-np-delete-main-5vq47-vdkbs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r2grpqzmpxnk--managed/providers/Microsoft.Compute/virtualMachines/r0n8s9h5n0s8u3c-np-delete-main-5vq47-vdkbs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000dfcbc0\u003e: \n failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-r2grpqzmpxnk\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2b31ee5-5378-49f4-b4f9-8c82f9bace93\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2b31ee5-5378-49f4-b4f9-8c82f9bace93\",\n \"name\": \"e2b31ee5-5378-49f4-b4f9-8c82f9bace93\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T13:59:14.617140907Z\",\n \"endTime\": \"2026-09-29T14:18:21.033986186Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine r0n8s9h5n0s8u3c-np-delete-main-5vq47-vdkbs: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r2grpqzmpxnk--managed/providers/Microsoft.Compute/virtualMachines/r0n8s9h5n0s8u3c-np-delete-main-5vq47-vdkbs to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine r0n8s9h5n0s8u3c-np-delete-main-5vq47-vdkbs: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-r2grpqzmpxnk--managed/providers/Microsoft.Compute/virtualMachines/r0n8s9h5n0s8u3c-np-delete-main-5vq47-vdkbs to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lwltp7hhv9xn\" to finish creating: GET https://management"}},{"row_id":"57d0f2c45ae5aa85fc0e32a05d627c5396db54ba3bc9fa5ca6ffe597f45469f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104910007010070528","occurred_at":"2026-09-29T12:23:47Z","failed_at":"e2e","signature_id":"27c362d1f8f08350e436cbafd13f1fab513c6ded4c6fd812584f5398a1817c87","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0014784b0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.003s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc0014784b0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0, targetMinor=5.1) failed: clusterversion status.history has no version in target minor \"5.1\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"588105c77b24bd2a5e7cd57d24d83c0ab20844e5f8e2d53af9ed218684bff895","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104910007010070528","occurred_at":"2026-09-29T12:23:47Z","failed_at":"e2e","signature_id":"0ca3511a0e612a034cebfee1d8cd276f31917dec0beaedfc729e0efe471fb74e","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c09f00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qds642pj6qdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8c7a098f-68ef-4837-bf5c-33fe66729db0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8c7a098f-68ef-4837-bf5c-33fe66729db0\",\n \"name\": \"8c7a098f-68ef-4837-bf5c-33fe66729db0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T12:58:44.310890224Z\",\n \"endTime\": \"2026-09-29T13:17:52.410585343Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x4e8d7k0u2h1t8g-np-1-4stht-tcn7q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qds642pj6qdc--managed/providers/Microsoft.Compute/virtualMachines/x4e8d7k0u2h1t8g-np-1-4stht-tcn7q to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x4e8d7k0u2h1t8g-np-1-4stht-tcn7q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qds642pj6qdc--managed/providers/Microsoft.Compute/virtualMachines/x4e8d7k0u2h1t8g-np-1-4stht-tcn7q to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c09f00\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qds642pj6qdc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8c7a098f-68ef-4837-bf5c-33fe66729db0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8c7a098f-68ef-4837-bf5c-33fe66729db0\",\n \"name\": \"8c7a098f-68ef-4837-bf5c-33fe66729db0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T12:58:44.310890224Z\",\n \"endTime\": \"2026-09-29T13:17:52.410585343Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine x4e8d7k0u2h1t8g-np-1-4stht-tcn7q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qds642pj6qdc--managed/providers/Microsoft.Compute/virtualMachines/x4e8d7k0u2h1t8g-np-1-4stht-tcn7q to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine x4e8d7k0u2h1t8g-np-1-4stht-tcn7q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-qds642pj6qdc--managed/providers/Microsoft.Compute/virtualMachines/x4e8d7k0u2h1t8g-np-1-4stht-tcn7q to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6337f3c67e6cffca410724e081c0b953d1e97d91961a26686250acd2ab6d952a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104910007010070528","occurred_at":"2026-09-29T12:23:47Z","failed_at":"e2e","signature_id":"59cc000d9a4b1040530948c554c649f3448d0a9df941d7f19e5197f158812295","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6lgs9c\" for cluster \"cluster-candidate-4-21-6lgs9c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aab760\u003e: \n failed to create NodePool np-6lgs9c: failed waiting for nodepool=\"np-6lgs9c\" for cluster \"cluster-candidate-4-21-6lgs9c\" in resourcegroup=\"rg-candidate-4-21-ptntl59g9fpg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c0c5da21-b394-4076-91ad-570f20be81a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c0c5da21-b394-4076-91ad-570f20be81a7\",\n \"name\": \"c0c5da21-b394-4076-91ad-570f20be81a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T13:23:06.981213639Z\",\n \"endTime\": \"2026-09-29T13:42:09.219679905Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p3t9k3i6s5x8k4x-np-6lgs9c-jwnkm-t8k7p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ptntl59g9fpg-candidate-6lgs9c--managed/providers/Microsoft.Compute/virtualMachines/p3t9k3i6s5x8k4x-np-6lgs9c-jwnkm-t8k7p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p3t9k3i6s5x8k4x-np-6lgs9c-jwnkm-t8k7p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ptntl59g9fpg-candidate-6lgs9c--managed/providers/Microsoft.Compute/virtualMachines/p3t9k3i6s5x8k4x-np-6lgs9c-jwnkm-t8k7p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-6lgs9c\" for cluster \"cluster-candidate-4-21-6lgs9c\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aab760\u003e: \n failed to create NodePool np-6lgs9c: failed waiting for nodepool=\"np-6lgs9c\" for cluster \"cluster-candidate-4-21-6lgs9c\" in resourcegroup=\"rg-candidate-4-21-ptntl59g9fpg\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c0c5da21-b394-4076-91ad-570f20be81a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/c0c5da21-b394-4076-91ad-570f20be81a7\",\n \"name\": \"c0c5da21-b394-4076-91ad-570f20be81a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T13:23:06.981213639Z\",\n \"endTime\": \"2026-09-29T13:42:09.219679905Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine p3t9k3i6s5x8k4x-np-6lgs9c-jwnkm-t8k7p: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ptntl59g9fpg-candidate-6lgs9c--managed/providers/Microsoft.Compute/virtualMachines/p3t9k3i6s5x8k4x-np-6lgs9c-jwnkm-t8k7p to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine p3t9k3i6s5x8k4x-np-6lgs9c-jwnkm-t8k7p: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-ptntl59g9fpg-candidate-6lgs9c--managed/providers/Microsoft.Compute/virtualMachines/p3t9k3i6s5x8k4x-np-6lgs9c-jwnkm-t8k7p to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104820270236700672","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T06:27:12Z","started_at":"2026-09-29T06:27:12Z","completed_at":"2026-09-29T09:02:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T09:10:52.898999145Z","timing_metadata_checked_at":"2026-09-29T09:10:52.898999145Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T09:10:52.902212535Z","region_metadata_checked_at":"2026-09-29T09:10:52.902212535Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9c1cca4f6dd8f3fff929f548c2a011f30ee25fbe908aa5016079394cedf69110","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104820270236700672","occurred_at":"2026-09-29T06:27:12Z","failed_at":"e2e","signature_id":"2be180833da00904f20d92e53243bf0c0ab95a701f93c3462ffafdef7d3e30a8","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001352fa0\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001352fa0\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred"}},{"row_id":"be517690a7ea60529b6258e872e9eb3c0e3eacae785ef01de502d64a599d41ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104820270236700672","occurred_at":"2026-09-29T06:27:12Z","failed_at":"e2e","signature_id":"9afab23b5f6b1315ffb8fb00c24eb4dd4f792a9d6614e70dc159ba3522949ab8","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:258]: failed to install cert-manager manifests\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b64160\u003e: \n GET https://github.com/cert-manager/cert-manager/releases/download/v1.20.2/cert-manager.yaml: HTTP 500\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:258]: failed to install cert-manager manifests\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000b64160\u003e: \n GET https://github.com/cert-manager/cert-manager/releases/download/v1.20.2/cert-manager.yaml: HTTP 500\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2db146a8f9f09087b07e04f5208d6fc12fe0e2ed29a2687bcf65d78dd048825","failure_pattern":"GET \u003curl\u003e","search_query":"GET https://github"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","f2db146a8f9f09087b07e04f5208d6fc12fe0e2ed29a2687bcf65d78dd048825"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104764681787805696","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T02:46:19Z","started_at":"2026-09-29T02:46:19Z","completed_at":"2026-09-29T05:12:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T05:29:31.280318302Z","timing_metadata_checked_at":"2026-09-29T05:29:31.280318302Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T05:29:31.283478191Z","region_metadata_checked_at":"2026-09-29T05:29:31.283478191Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104753137003794432","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T02:00:26Z","started_at":"2026-09-29T02:00:26Z","completed_at":"2026-09-29T04:08:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:17:04.661440538Z","timing_metadata_checked_at":"2026-09-29T04:17:04.661440538Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:17:04.664929825Z","region_metadata_checked_at":"2026-09-29T04:17:04.664929825Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"c1e800ff9bcd7b6d6e768d689bbcd1814b34d78442d743e72a15f7b98a975755","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104753137003794432","occurred_at":"2026-09-29T02:00:26Z","failed_at":"e2e","signature_id":"b89d6be6f9aa2985f1399ee4c1c85e832179545db38287fad87336488a68148b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fktv4d\" for cluster \"cluster-candidate-4-21-fktv4d\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010608e0\u003e: \n failed to create NodePool np-fktv4d: failed waiting for nodepool=\"np-fktv4d\" for cluster \"cluster-candidate-4-21-fktv4d\" in resourcegroup=\"rg-candidate-4-21-hd26bdgtn7gq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01d13c95-d010-4354-af72-1e3f0c8961b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01d13c95-d010-4354-af72-1e3f0c8961b4\",\n \"name\": \"01d13c95-d010-4354-af72-1e3f0c8961b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:40:40.344373047Z\",\n \"endTime\": \"2026-09-29T02:59:45.460904082Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e7o6n1m5a6l5q6f-np-fktv4d-k4bkc-pvr2z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hd26bdgtn7gq-candidate-fktv4d--managed/providers/Microsoft.Compute/virtualMachines/e7o6n1m5a6l5q6f-np-fktv4d-k4bkc-pvr2z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e7o6n1m5a6l5q6f-np-fktv4d-k4bkc-pvr2z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hd26bdgtn7gq-candidate-fktv4d--managed/providers/Microsoft.Compute/virtualMachines/e7o6n1m5a6l5q6f-np-fktv4d-k4bkc-pvr2z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fktv4d\" for cluster \"cluster-candidate-4-21-fktv4d\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010608e0\u003e: \n failed to create NodePool np-fktv4d: failed waiting for nodepool=\"np-fktv4d\" for cluster \"cluster-candidate-4-21-fktv4d\" in resourcegroup=\"rg-candidate-4-21-hd26bdgtn7gq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01d13c95-d010-4354-af72-1e3f0c8961b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/01d13c95-d010-4354-af72-1e3f0c8961b4\",\n \"name\": \"01d13c95-d010-4354-af72-1e3f0c8961b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:40:40.344373047Z\",\n \"endTime\": \"2026-09-29T02:59:45.460904082Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e7o6n1m5a6l5q6f-np-fktv4d-k4bkc-pvr2z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hd26bdgtn7gq-candidate-fktv4d--managed/providers/Microsoft.Compute/virtualMachines/e7o6n1m5a6l5q6f-np-fktv4d-k4bkc-pvr2z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e7o6n1m5a6l5q6f-np-fktv4d-k4bkc-pvr2z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-hd26bdgtn7gq-candidate-fktv4d--managed/providers/Microsoft.Compute/virtualMachines/e7o6n1m5a6l5q6f-np-fktv4d-k4bkc-pvr2z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d9033ac74005faba8bd82ed99db953dacf982b3d0fd844e315c50556f215523d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104753137003794432","occurred_at":"2026-09-29T02:00:26Z","failed_at":"e2e","signature_id":"33621102dea1edffd8d9d82888ca76dfd48a2133799de23ef4c4ceb0557ccf33","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011bb80\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-br9bvcrqkdrx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/267eabaf-73d5-414d-b422-0156f94fa98b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/267eabaf-73d5-414d-b422-0156f94fa98b\",\n \"name\": \"267eabaf-73d5-414d-b422-0156f94fa98b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:06.808479674Z\",\n \"endTime\": \"2026-09-29T03:19:12.619464984Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-9q5vb-nv244: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-br9bvcrqkdrx--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-9q5vb-nv244 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-9q5vb-nv244: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-br9bvcrqkdrx--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-9q5vb-nv244 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011bb80\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-br9bvcrqkdrx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/267eabaf-73d5-414d-b422-0156f94fa98b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/267eabaf-73d5-414d-b422-0156f94fa98b\",\n \"name\": \"267eabaf-73d5-414d-b422-0156f94fa98b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T03:00:06.808479674Z\",\n \"endTime\": \"2026-09-29T03:19:12.619464984Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-9q5vb-nv244: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-br9bvcrqkdrx--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-9q5vb-nv244 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-9q5vb-nv244: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-br9bvcrqkdrx--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-9q5vb-nv244 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e6a58924ff2b8cd9a1610ec21e9ed2b4168abb92fc879f04811550abcb1f42ac","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104753137003794432","occurred_at":"2026-09-29T02:00:26Z","failed_at":"e2e","signature_id":"d21e0707f48811391cdfcaec8243cf71d75df1c8a221e90daf8c7d60467d92de","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083c440\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-j2w7rvh22rc8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d195bba-0850-461e-a119-0cb340343650\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d195bba-0850-461e-a119-0cb340343650\",\n \"name\": \"9d195bba-0850-461e-a119-0cb340343650\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:16:42.180371111Z\",\n \"endTime\": \"2026-09-29T02:35:43.192706201Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5l6m9s6s7e2g7d-np-1-sktsn-kwql2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-j2w7rvh22rc8--managed/providers/Microsoft.Compute/virtualMachines/s5l6m9s6s7e2g7d-np-1-sktsn-kwql2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5l6m9s6s7e2g7d-np-1-sktsn-kwql2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-j2w7rvh22rc8--managed/providers/Microsoft.Compute/virtualMachines/s5l6m9s6s7e2g7d-np-1-sktsn-kwql2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083c440\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-j2w7rvh22rc8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d195bba-0850-461e-a119-0cb340343650\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9d195bba-0850-461e-a119-0cb340343650\",\n \"name\": \"9d195bba-0850-461e-a119-0cb340343650\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:16:42.180371111Z\",\n \"endTime\": \"2026-09-29T02:35:43.192706201Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s5l6m9s6s7e2g7d-np-1-sktsn-kwql2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-j2w7rvh22rc8--managed/providers/Microsoft.Compute/virtualMachines/s5l6m9s6s7e2g7d-np-1-sktsn-kwql2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s5l6m9s6s7e2g7d-np-1-sktsn-kwql2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-j2w7rvh22rc8--managed/providers/Microsoft.Compute/virtualMachines/s5l6m9s6s7e2g7d-np-1-sktsn-kwql2 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104751055710457856","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T01:52:10Z","started_at":"2026-09-29T01:52:10Z","completed_at":"2026-09-29T04:06:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:17:04.640889412Z","timing_metadata_checked_at":"2026-09-29T04:17:04.640889412Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:17:04.643786901Z","region_metadata_checked_at":"2026-09-29T04:17:04.643786901Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"89e7922b6013f64991d468a198c1e2e20ff317451ca60e6fa94562678af927c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104751055710457856","occurred_at":"2026-09-29T01:52:10Z","failed_at":"e2e","signature_id":"910b267959d06218fce4c3fcdcdb426155e3fd6043ab29142074547538c5b774","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001005800\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-w2rs2wktq5hp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/78d682cc-8178-44fc-8481-457b338bfc70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/78d682cc-8178-44fc-8481-457b338bfc70\",\n \"name\": \"78d682cc-8178-44fc-8481-457b338bfc70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:05:25.77800873Z\",\n \"endTime\": \"2026-09-29T02:24:26.035091452Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a0a1s6r0g7y9m9a-nodepool-128gib-bkb4m-hkxx5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-w2rs2wktq5hp--managed/providers/Microsoft.Compute/virtualMachines/a0a1s6r0g7y9m9a-nodepool-128gib-bkb4m-hkxx5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a0a1s6r0g7y9m9a-nodepool-128gib-bkb4m-hkxx5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-w2rs2wktq5hp--managed/providers/Microsoft.Compute/virtualMachines/a0a1s6r0g7y9m9a-nodepool-128gib-bkb4m-hkxx5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001005800\u003e: \n failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-w2rs2wktq5hp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/78d682cc-8178-44fc-8481-457b338bfc70\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/78d682cc-8178-44fc-8481-457b338bfc70\",\n \"name\": \"78d682cc-8178-44fc-8481-457b338bfc70\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:05:25.77800873Z\",\n \"endTime\": \"2026-09-29T02:24:26.035091452Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a0a1s6r0g7y9m9a-nodepool-128gib-bkb4m-hkxx5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-w2rs2wktq5hp--managed/providers/Microsoft.Compute/virtualMachines/a0a1s6r0g7y9m9a-nodepool-128gib-bkb4m-hkxx5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a0a1s6r0g7y9m9a-nodepool-128gib-bkb4m-hkxx5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-w2rs2wktq5hp--managed/providers/Microsoft.Compute/virtualMachines/a0a1s6r0g7y9m9a-nodepool-128gib-bkb4m-hkxx5 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"stg","summary":{"total_runs":3,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T23:00:27Z","started_at":"2026-09-29T23:00:27Z","completed_at":"2026-09-30T00:40:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-30T00:53:48.441211888Z","timing_metadata_checked_at":"2026-09-30T00:53:48.441211888Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-30T00:53:48.44387648Z","region_metadata_checked_at":"2026-09-30T00:53:48.44387648Z"},"failed_at":["e2e"],"failed_test_count":10,"occurrences":[{"row_id":"0ed1b81fea59a83c513494025bec0f5a4d6d38f2edbd770674cbae16597ad75d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"8eed6506018065789817fe9b51be37835e1602d5b7dfa00414172144091193bd","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000910960\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-7fhm6z\" in resourcegroup=\"rg-np-skip-minor-7fhm6z-prdlxbznjw2d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-7fhm6z-prdlxbznjw2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-7fhm6z/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000910960\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-7fhm6z\" in resourcegroup=\"rg-np-skip-minor-7fhm6z-prdlxbznjw2d\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-7fhm6z-prdlxbznjw2d/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-7fhm6z/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"0f56aeb0a13235e6cdcb248fa6593c0820566e3af59bffcd427e99ad9f90bdd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"60659d0f9c7d63dac88f402352f588c887060d9b24a3463977fc76342780c9e6","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000895700\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-6nhc5q\" resourcegroup=\"rg-np-dg-minor-6nhc5q-8fsxplbkq2t5\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-6nhc5q-8fsxplbkq2t5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-6nhc5q/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000895700\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-6nhc5q\" resourcegroup=\"rg-np-dg-minor-6nhc5q-8fsxplbkq2t5\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-6nhc5q-8fsxplbkq2t5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-6nhc5q/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"1005ae96b484e95285fba7cc1733865e36f37bc7e47aaf3d03109fc240829126","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"8e31a23f2d6d08760b453af93ef1e43da2ce2ee3a90fc638a9681e25e0480a99","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001684150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-qccnxb6p8bvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc001684150\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-qccnxb6p8bvl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8b334e65b0c6a4f286c5d073544e4f4611fae9942eec3d05976f08a6d7482787","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"5b7809cef93cf7abba12b1eafdaf1df26222b2bc0718cfce631ffc2837fdda64","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pvgxzh\" for cluster \"cluster-nightly-4-20-pvgxzh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010d80e0\u003e: \n failed to create NodePool np-pvgxzh: failed starting nodepool creation \"np-pvgxzh\" for cluster \"cluster-nightly-4-20-pvgxzh\" in resourcegroup=\"rg-nightly-4-20-p6d2fjzl4q89\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-p6d2fjzl4q89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pvgxzh/nodePools/np-pvgxzh\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-pvgxzh\" for cluster \"cluster-nightly-4-20-pvgxzh\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010d80e0\u003e: \n failed to create NodePool np-pvgxzh: failed starting nodepool creation \"np-pvgxzh\" for cluster \"cluster-nightly-4-20-pvgxzh\" in resourcegroup=\"rg-nightly-4-20-p6d2fjzl4q89\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-p6d2fjzl4q89/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-pvgxzh/nodePools/np-pvgxzh\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a651b0d6f2349cf81860e92293f5c0c21483a3fc4d91feec21e9ee5c0add47f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"8ac6227b148ca4e7318af763502869b17d4ec721b6b6ab3459104f673e6b5e3c","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00150c9c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2s6gqzlqnmd9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00150c9c0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-2s6gqzlqnmd9/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b2fc52aa284f2091fcdaaee2cfbbf367bf590b2065dbdf6e946a125d910a9392","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"f042798528e77b8aacc05802f53c481964a6f05c68be2801c3c03228c69d0694","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ad57a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hwqlx7qffxrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000ad57a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hwqlx7qffxrk/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bc202cd006c68cd6e8c22270fd8e3f80bcc3ebe0d7a5b5850ca0a39c1364eee1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"382ad050b070be18decd763e09c32351650cccdd4813dc1a2fcce71eaf771746","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e90e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gfr8dlt6lb7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008e90e0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-gfr8dlt6lb7m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"eddb137379e4834023ee200107c4038ebc671fe6b2c7aa9862fb6f0242b8518d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"00a6fee9e5d52d97c57b92fba12a81e4cff474e82f5bf83399148cc563c1d63c","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097e440\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-kqcc66\" in resourcegroup=\"rg-np-version-upgrade-kqcc66-8mp5bjlqdlxf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kqcc66-8mp5bjlqdlxf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-kqcc66/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00097e440\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-kqcc66\" in resourcegroup=\"rg-np-version-upgrade-kqcc66-8mp5bjlqdlxf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-kqcc66-8mp5bjlqdlxf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-kqcc66/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f5cf3a0e6f50966af4778db533d905fffa836b009ee186a06a6330a219f006d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"d35c0de4ee6ff29dd14eaac43fabeda69fd23c610be39aea8740dd9320fe573c","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00177c4e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bd8vrt2phbnw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00177c4e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-bd8vrt2phbnw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f780445daff9c89fec90569c237e200711c3c314e1a971b9915b30b37740b810","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2105070230005878784","occurred_at":"2026-09-29T23:00:27Z","failed_at":"e2e","signature_id":"28d6214fe9cae61f9c21b48ef09821799d67ddf61d24258fc31f37b7d099327a","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00131c360\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-29-110018, targetMinor=5.1.0-0.nightly-multi-2026-09-29-141926) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-29-141926\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.004s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc00131c360\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-29-110018, targetMinor=5.1.0-0.nightly-multi-2026-09-29-141926) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-29-141926\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}}],"failure_pattern_summary":{"signature_count":10,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f"],"matched_occurrences":10,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2104809159978389504","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-29T05:43:03Z","started_at":"2026-09-29T05:43:03Z","completed_at":"2026-09-29T07:58:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T08:07:28.394290691Z","timing_metadata_checked_at":"2026-09-29T08:07:28.394290691Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T08:07:28.397082982Z","region_metadata_checked_at":"2026-09-29T08:07:28.397082982Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2104753137033154560","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-29T02:00:26Z","started_at":"2026-09-29T02:00:26Z","completed_at":"2026-09-29T03:55:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T04:06:57.948588323Z","timing_metadata_checked_at":"2026-09-29T04:06:57.948588323Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T04:06:57.952178711Z","region_metadata_checked_at":"2026-09-29T04:06:57.952178711Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"484c737261b633e5d6ebe993daf89855579c79c839448ab3d3d1522ff1fd01c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2104753137033154560","occurred_at":"2026-09-29T02:00:26Z","failed_at":"e2e","signature_id":"93d8d3b68ea20f1669964c686519fb7188342e904b547b2d52c60154628dd90e","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010098a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-2x9ww8xf75tr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2ca4ae6-8c7b-4757-944a-a1ada0f17a94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2ca4ae6-8c7b-4757-944a-a1ada0f17a94\",\n \"name\": \"e2ca4ae6-8c7b-4757-944a-a1ada0f17a94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:12:36.53948617Z\",\n \"endTime\": \"2026-09-29T02:31:39.690266696Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d5a6t5l1x4b3i2w-np-1-wqsmx-cskns: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-2x9ww8xf75tr--managed/providers/Microsoft.Compute/virtualMachines/d5a6t5l1x4b3i2w-np-1-wqsmx-cskns to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d5a6t5l1x4b3i2w-np-1-wqsmx-cskns: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-2x9ww8xf75tr--managed/providers/Microsoft.Compute/virtualMachines/d5a6t5l1x4b3i2w-np-1-wqsmx-cskns to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010098a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-2x9ww8xf75tr\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2ca4ae6-8c7b-4757-944a-a1ada0f17a94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e2ca4ae6-8c7b-4757-944a-a1ada0f17a94\",\n \"name\": \"e2ca4ae6-8c7b-4757-944a-a1ada0f17a94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T02:12:36.53948617Z\",\n \"endTime\": \"2026-09-29T02:31:39.690266696Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d5a6t5l1x4b3i2w-np-1-wqsmx-cskns: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-2x9ww8xf75tr--managed/providers/Microsoft.Compute/virtualMachines/d5a6t5l1x4b3i2w-np-1-wqsmx-cskns to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d5a6t5l1x4b3i2w-np-1-wqsmx-cskns: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cx-rg-cluster-2x9ww8xf75tr--managed/providers/Microsoft.Compute/virtualMachines/d5a6t5l1x4b3i2w-np-1-wqsmx-cskns to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]}]}