{"meta":{"date":"2026-09-28","timezone":"UTC","generated_at":"2026-10-14T15:27:46Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":114,"failed_runs":97,"runs_with_occurrences":97,"runs_with_failure_pattern_matches":87,"runs_with_unmatched_occurrences":10,"runs_without_occurrence_rows":17,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104719685969776640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"closed","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T23:47:31Z","started_at":"2026-09-28T23:47:31Z","completed_at":"2026-09-29T00:07:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T00:21:20.751061835Z","timing_metadata_checked_at":"2026-09-29T00:21:20.751061835Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-29T00:21:20.754161925Z","region_metadata_checked_at":"2026-09-29T01:25:17.612444152Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b57cf10ad956b9eb0f7a44d0f73d69c01853e1a08950e3d3b211ed62978712a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104719685969776640","occurred_at":"2026-09-28T23:47:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104712961779044352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"26379bcf454e0b78f2bfb84abc8a9ce0a11afdcb","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-28T23:20:48Z","started_at":"2026-09-28T23:20:48Z","completed_at":"2026-09-29T01:32:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:45:25.43098363Z","timing_metadata_checked_at":"2026-09-29T01:45:25.43098363Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:45:25.434474019Z","region_metadata_checked_at":"2026-09-29T01:45:25.434474019Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7235/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104711495257427968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7235,"pr_state":"closed","pr_sha":"6494047d3fea361fc5996e9eadf7e2c60f551ea8","final_merged_sha":"6494047d3fea361fc5996e9eadf7e2c60f551ea8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T23:14:58Z","started_at":"2026-09-28T23:14:58Z","completed_at":"2026-09-29T01:25:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:35:23.689651308Z","timing_metadata_checked_at":"2026-09-29T01:35:23.689651308Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:35:23.692466399Z","region_metadata_checked_at":"2026-09-29T01:35:23.692466399Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7234/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104711131258949632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7234,"pr_state":"closed","pr_sha":"b4d7881116fc22565f42759a0932e5c7eeaf4d59","final_merged_sha":"b4d7881116fc22565f42759a0932e5c7eeaf4d59","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T23:13:31Z","started_at":"2026-09-28T23:13:31Z","completed_at":"2026-09-29T01:31:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:45:25.812297669Z","timing_metadata_checked_at":"2026-09-29T01:45:25.812297669Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:45:25.81495816Z","region_metadata_checked_at":"2026-09-29T01:45:25.81495816Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7233/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104709786435063808","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7233,"pr_state":"open","pr_sha":"b93b8c67088465e0f9e9154753b80f60071d5f91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T23:08:11Z","started_at":"2026-09-28T23:08:11Z","completed_at":"2026-09-29T01:16:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:25:17.802309579Z","timing_metadata_checked_at":"2026-09-29T01:25:17.802309579Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:25:17.805254369Z","region_metadata_checked_at":"2026-09-29T01:25:17.805254369Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"446fb8b104a9550376a2f33b1bfd511c50fb5cf606d860a213f8367404f2d855","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7233/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104709786435063808","occurred_at":"2026-09-28T23:08:11Z","failed_at":"e2e","signature_id":"ca1052bcf33487a021334893241ae8188017185f1642b6a294d2994f137f4da1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-m8jhjs\" for cluster \"cluster-candidate-4-23-m8jhjs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86d20\u003e: \n failed to create NodePool np-m8jhjs: failed waiting for nodepool=\"np-m8jhjs\" for cluster \"cluster-candidate-4-23-m8jhjs\" in resourcegroup=\"rg-candidate-4-23-f6rffb7vgk7k\" to finish creating: GET https://rp.j5063808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/451c3970-ee1e-443a-8418-d00eafee945e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/451c3970-ee1e-443a-8418-d00eafee945e\",\n \"name\": \"451c3970-ee1e-443a-8418-d00eafee945e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:03:14.031479379Z\",\n \"endTime\": \"2026-09-29T00:22:18.567634171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-f6rffb7vgk7k-candidate-m8jhjs--managed/providers/Microsoft.Compute/virtualMachines/q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-f6rffb7vgk7k-candidate-m8jhjs--managed/providers/Microsoft.Compute/virtualMachines/q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-m8jhjs\" for cluster \"cluster-candidate-4-23-m8jhjs\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86d20\u003e: \n failed to create NodePool np-m8jhjs: failed waiting for nodepool=\"np-m8jhjs\" for cluster \"cluster-candidate-4-23-m8jhjs\" in resourcegroup=\"rg-candidate-4-23-f6rffb7vgk7k\" to finish creating: GET https://rp.j5063808.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/451c3970-ee1e-443a-8418-d00eafee945e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/451c3970-ee1e-443a-8418-d00eafee945e\",\n \"name\": \"451c3970-ee1e-443a-8418-d00eafee945e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:03:14.031479379Z\",\n \"endTime\": \"2026-09-29T00:22:18.567634171Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-f6rffb7vgk7k-candidate-m8jhjs--managed/providers/Microsoft.Compute/virtualMachines/q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-23-f6rffb7vgk7k-candidate-m8jhjs--managed/providers/Microsoft.Compute/virtualMachines/q4i8o9w4d0x3l0c-np-m8jhjs-4k5ks-9g9vh to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104709623964504064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7124,"pr_state":"closed","pr_sha":"c8ac987d56f8213ad9e264daec91005828041ca1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T23:07:32Z","started_at":"2026-09-28T23:07:32Z","completed_at":"2026-09-29T01:26:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:35:23.375708346Z","timing_metadata_checked_at":"2026-09-29T01:35:23.375708346Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:35:23.378944835Z","region_metadata_checked_at":"2026-09-29T01:35:23.378944835Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"aaf6f5ea803d02db9f4bd71a71139b3458075920b66a45b0cac43a13c635e3c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104709623964504064","occurred_at":"2026-09-28T23:07:32Z","failed_at":"alert","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-29T00:14:18Z\n Ended: 2026-09-29T00:26:21Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j4504064-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104709118777364480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7214,"pr_state":"closed","pr_sha":"2ae778845f66d98316f4f41951553bf09a1e19db","final_merged_sha":"2ae778845f66d98316f4f41951553bf09a1e19db","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T23:05:32Z","started_at":"2026-09-28T23:05:32Z","completed_at":"2026-09-29T01:20:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:35:23.499996535Z","timing_metadata_checked_at":"2026-09-29T01:35:23.499996535Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:35:23.502643126Z","region_metadata_checked_at":"2026-09-29T01:35:23.502643126Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104702432968708096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T22:38:58Z","started_at":"2026-09-28T22:38:58Z","completed_at":"2026-09-29T00:54:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:04:03.445454925Z","timing_metadata_checked_at":"2026-09-29T01:04:03.445454925Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:04:03.449182313Z","region_metadata_checked_at":"2026-09-29T01:04:03.449182313Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0949316e35b997950b31193bff5af37a2f1e66a2d589869c12a542b73d25e11d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104702432968708096","occurred_at":"2026-09-28T22:38:58Z","failed_at":"e2e","signature_id":"98e7af2c661ff33cf1fa2bc30a96c5bfa7c31b06b9aa1b278fa59c5194434907","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff7440\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rdwv792csbm5\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf2e602a-624e-40d9-b1d9-4619bec9a257\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf2e602a-624e-40d9-b1d9-4619bec9a257\",\n \"name\": \"bf2e602a-624e-40d9-b1d9-4619bec9a257\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:36:16.368959757Z\",\n \"endTime\": \"2026-09-28T23:55:24.990728156Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b8s9z5t1h6u2q2n-np-1-b7tcg-zs7fm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rdwv792csbm5--managed/providers/Microsoft.Compute/virtualMachines/b8s9z5t1h6u2q2n-np-1-b7tcg-zs7fm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8s9z5t1h6u2q2n-np-1-b7tcg-d9rrr: Failed: availabilitysets failed to create or update. err: failed to get existing resource fips-enabled-rdwv792csbm5--managed/2t4ak80phdi6nre3sgcdj1b7rfktbt35_b8s9z5t1h6u2q2n-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rdwv792csbm5--managed/providers/Microsoft.Compute/availabilitySets/2t4ak80phdi6nre3sgcdj1b7rfktbt35_b8s9z5t1h6u2q2n-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '1' seconds a... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ff7440\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rdwv792csbm5\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf2e602a-624e-40d9-b1d9-4619bec9a257\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf2e602a-624e-40d9-b1d9-4619bec9a257\",\n \"name\": \"bf2e602a-624e-40d9-b1d9-4619bec9a257\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:36:16.368959757Z\",\n \"endTime\": \"2026-09-28T23:55:24.990728156Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b8s9z5t1h6u2q2n-np-1-b7tcg-zs7fm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rdwv792csbm5--managed/providers/Microsoft.Compute/virtualMachines/b8s9z5t1h6u2q2n-np-1-b7tcg-zs7fm to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b8s9z5t1h6u2q2n-np-1-b7tcg-d9rrr: Failed: availabilitysets failed to create or update. err: failed to get existing resource fips-enabled-rdwv792csbm5--managed/2t4ak80phdi6nre3sgcdj1b7rfktbt35_b8s9z5t1h6u2q2n-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rdwv792csbm5--managed/providers/Microsoft.Compute/availabilitySets/2t4ak80phdi6nre3sgcdj1b7rfktbt35_b8s9z5t1h6u2q2n-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3015be55-a361-4a86-8439-0abc7860a4ef' exceeded. Please try again after '1' seconds a... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"bdc3f2a5dfb44ece65847fa574538feb01aa4a19bbeeae5c8aa4f7835b051b9c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104702432968708096","occurred_at":"2026-09-28T22:38:58Z","failed_at":"e2e","signature_id":"e75e1ad4b0432801e2acb69e7608842fb32bc692d9078a7d6cfc759e2f16fd9a","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:247]: failed to create node pool\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000687000\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull-create\" in resourcegroup=\"acr-pull-create-vdlktwxnm8vh\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fb52b81-388d-4086-8055-43b274743b11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fb52b81-388d-4086-8055-43b274743b11\",\n \"name\": \"9fb52b81-388d-4086-8055-43b274743b11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:41:09.989503835Z\",\n \"endTime\": \"2026-09-28T23:41:15.954914297Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:247]: failed to create node pool\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000687000\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"acr-pull-create\" in resourcegroup=\"acr-pull-create-vdlktwxnm8vh\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fb52b81-388d-4086-8055-43b274743b11\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9fb52b81-388d-4086-8055-43b274743b11\",\n \"name\": \"9fb52b81-388d-4086-8055-43b274743b11\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:41:09.989503835Z\",\n \"endTime\": \"2026-09-28T23:41:15.954914297Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39","failure_pattern":"ERROR CODE: InternalServerError; detail message [clusterServiceNodePoolStatus] failed to create transaction: pq: sorry, too many clients already; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c1bf0305fcecd817e7c64f2e68ba13e0209fb65db32d3369f1837aa959f12ba5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104702432968708096","occurred_at":"2026-09-28T22:38:58Z","failed_at":"e2e","signature_id":"7a7e77c690889a19cb56e1f50e820d8f0f557b662a6c09c58c3793ba99c49cd2","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b234e0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-nl2nr6\" in resourcegroup=\"rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af93dcf2-82e0-4298-856d-f4846917b52e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af93dcf2-82e0-4298-856d-f4846917b52e\",\n \"name\": \"af93dcf2-82e0-4298-856d-f4846917b52e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:46:23.744577199Z\",\n \"endTime\": \"2026-09-29T00:05:30.871348708Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn-np-upgrade-nl-3d9177ea/providers/Microsoft.Compute/virtualMachines/m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn-np-upgrade-nl-3d9177ea/providers/Microsoft.Compute/virtualMachines/m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b234e0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-nl2nr6\" in resourcegroup=\"rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn\" to finish creating: GET https://rp.j8708096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af93dcf2-82e0-4298-856d-f4846917b52e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/af93dcf2-82e0-4298-856d-f4846917b52e\",\n \"name\": \"af93dcf2-82e0-4298-856d-f4846917b52e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:46:23.744577199Z\",\n \"endTime\": \"2026-09-29T00:05:30.871348708Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn-np-upgrade-nl-3d9177ea/providers/Microsoft.Compute/virtualMachines/m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-nl2nr6-v9d65mgcz4rn-np-upgrade-nl-3d9177ea/providers/Microsoft.Compute/virtualMachines/m4b5p7t4y3f1r7l-npupgrade-4-20-glfwm-jbx5z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","fe24c8b4b28d3d2b1937803a92704c44ec574f3d1c3b740ec6d97282a1e18d39"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104701088555864064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7231,"pr_state":"open","pr_sha":"9c031e143321efd4a56c5aa92014227566ddf901","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T22:33:37Z","started_at":"2026-09-28T22:33:37Z","completed_at":"2026-09-29T00:58:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:14:27.922655344Z","timing_metadata_checked_at":"2026-09-29T01:14:27.922655344Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:14:27.925933233Z","region_metadata_checked_at":"2026-09-29T01:14:27.925933233Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8a11ee8cfdc73c694fe9d7e209ec6194feb7912e3b9fd1ad6fec233c43ab8429","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7231/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104701088555864064","occurred_at":"2026-09-28T22:33:37Z","failed_at":"e2e","signature_id":"4eb025693d04a56b1a1cdecd99f4ccf2153a73ec54e9bcd017b4a02505bf1b0f","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001664240\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:140]: failed to verify cluster health or IngressController scope for cluster \"private-ingress\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001664240\u003e: \n VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996","failure_pattern":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil","search_query":"VerifyIngressControllerScope(Internal) failed: IngressController loadBalancer config is nil"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a65699c58f94a1d1e9379f0751c4b9cac8b2c4a2d3b4115443123d6820743996"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104696906683781120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"035492034053829eae92f2b80d7ea0637ca6a0ba","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T22:17:00Z","started_at":"2026-09-28T22:17:00Z","completed_at":"2026-09-28T23:00:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T23:07:16.759928498Z","timing_metadata_checked_at":"2026-09-28T23:07:16.759928498Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T23:07:16.763052387Z","region_metadata_checked_at":"2026-09-28T23:07:16.763052387Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"03f29023bcfba1e3b6e7643d334cc3c8e90985ddf689f39702a6104e8fefe562","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104696906683781120","occurred_at":"2026-09-28T22:17:00Z","failed_at":"provision","signature_id":"7f499ac06c32b370e2096e3b168803976b54da2a1f11dc54f9c1679c6f5bae1f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T22:45:44.302Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T22:53:20.758Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T22:45:44.312984171Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T22:45:46.293945727Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T22:45:46.372370612Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755330702Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"93D44669EC2FD152\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:50:52.9549448Z\\\",\\\"duration\\\":\\\"PT2M24.7661361S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755440055Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"D5BB7C72C0AE0DC5\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:50:52.9549448Z\\\",\\\"duration\\\":\\\"PT2M26.0225935S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755461175Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"3C41555BEADB3DE5\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:45:49.7869316Z\\\",\\\"duration\\\":\\\"PT6M25.5435412S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755483916Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E73090DFF813B1FF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:50:52.9549448Z\\\",\\\"duration\\\":\\\"PT1M41.3605417S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755497506Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DB7E7572BE861E73\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T22:50:52.9549448Z\\\",\\\"duration\\\":\\\"PT1M37.1165773S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25934385-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25934385-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755594868Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A1922F5B4A5E438F\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T22:45:49.7869316Z\\\",\\\"duration\\\":\\\"PT5M2.9744241S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T22:53:20.755630539Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3781120-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3781120-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109713389587325\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3781120-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25934385-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3781120-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-25934385-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m36.442652668s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104682429376630784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"closed","pr_sha":"6dc92c0e1eae82818633321751269a6063e49fa2","final_merged_sha":"6dc92c0e1eae82818633321751269a6063e49fa2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T21:19:23Z","started_at":"2026-09-28T21:19:23Z","completed_at":"2026-09-28T23:30:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T23:39:18.378012829Z","timing_metadata_checked_at":"2026-09-28T23:39:18.378012829Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T23:39:18.381721218Z","region_metadata_checked_at":"2026-09-28T23:39:18.381721218Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104678718394863616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-28T21:04:44Z","started_at":"2026-09-28T21:04:44Z","completed_at":"2026-09-28T23:22:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T23:39:18.748057478Z","timing_metadata_checked_at":"2026-09-28T23:39:18.748057478Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T23:39:18.751801567Z","region_metadata_checked_at":"2026-09-28T23:39:18.751801567Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104671220124356608","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6915,"pr_state":"open","pr_sha":"1db778125f2b121f01dc06536c9f02dc6ca0aa21","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-28T20:34:56Z","started_at":"2026-09-28T20:34:56Z","completed_at":"2026-09-28T22:48:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T22:56:27.316379852Z","timing_metadata_checked_at":"2026-09-28T22:56:27.316379852Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T22:56:27.319534942Z","region_metadata_checked_at":"2026-09-28T22:56:27.319534942Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104669678952517632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7229,"pr_state":"closed","pr_sha":"71e7638a43793407ad32e40630f0c20d19617a22","final_merged_sha":"e251feee783228a94dd7ed8590b63c06c237cbb6","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T20:28:48Z","started_at":"2026-09-28T20:28:48Z","completed_at":"2026-09-28T22:35:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T22:46:16.683783447Z","timing_metadata_checked_at":"2026-09-28T22:46:16.683783447Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T22:46:16.687135236Z","region_metadata_checked_at":"2026-09-28T22:46:16.687135236Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"50b531f2b99e73a4dc0a2699f319d0ef5b01266c1ef5286e2529865af59bf8fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104669678952517632","occurred_at":"2026-09-28T20:28:48Z","failed_at":"e2e","signature_id":"bfb0b49b1e98aa126ca5cf25522dd823c2ee08043d22226794dfcfc1bbb69bcd","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012aa3f0\u003e: \n GET https://rp.j2517632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df74e584-612a-4664-98e5-95894a223dd0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df74e584-612a-4664-98e5-95894a223dd0\",\n \"name\": \"df74e584-612a-4664-98e5-95894a223dd0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T21:13:31.817925786Z\",\n \"endTime\": \"2026-09-28T21:42:34.60101187Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/clusters_sharing_resgroup.go:138]: failed to wait for second cluster \"basic-hcp-cluster2\" to complete creation (timeout '30.000000' minutes)\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0012aa3f0\u003e: \n GET https://rp.j2517632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df74e584-612a-4664-98e5-95894a223dd0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/df74e584-612a-4664-98e5-95894a223dd0\",\n \"name\": \"df74e584-612a-4664-98e5-95894a223dd0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T21:13:31.817925786Z\",\n \"endTime\": \"2026-09-28T21:42:34.60101187Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"801633e79b272b959410351c40c7b3945b87cc62348dddca0566fafb636cc623","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104669678952517632","occurred_at":"2026-09-28T20:28:48Z","failed_at":"alert","signature_id":"e0f653efbaa66eeb75116383c05ff2fa3c1601a49c7ad496cd303b7011f3952a","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T21:47:52Z\n Ended: 2026-09-28T22:02:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2517632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2517632/providers/microsoft.monitor/accounts/hcps-j2517632\", namespace=\"ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2xkdh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"a8dd6895d64a03e854d063376c2a70d7f25d710907f538bb32c301487ea007b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7229/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104669678952517632","occurred_at":"2026-09-28T20:28:48Z","failed_at":"alert","signature_id":"cb695e52ed5480bf4c59e72c73a827f71f03695861ee7b5fcc624983d0793694","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T21:44:52Z\n Ended: 2026-09-28T22:03:53Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2517632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2517632/providers/microsoft.monitor/accounts/hcps-j2517632\", namespace=\"ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2xkdh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T21:44:52Z\n Ended: 2026-09-28T22:00:53Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j2517632-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2517632/providers/microsoft.monitor/accounts/hcps-j2517632\", namespace=\"ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2xkdh\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2t48kv9d505es3p8ivbsahgh3k9orrao-z6g9i6m9w7w8g8g/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104648687849312256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T19:05:24Z","started_at":"2026-09-28T19:05:24Z","completed_at":"2026-09-28T21:16:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T21:32:16.070026884Z","timing_metadata_checked_at":"2026-09-28T21:32:16.070026884Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T21:32:16.073148074Z","region_metadata_checked_at":"2026-09-28T21:32:16.073148074Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a13f85133933adb088f0d4709c69af69b247a0fa11bd11146ccbec0b9fdcbe26","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104648687849312256","occurred_at":"2026-09-28T19:05:24Z","failed_at":"alert","signature_id":"5ec40f1dbe64ce284597d9fca92af72b5138aaab20deba744a9e8c7388175941","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T20:22:39Z\n Ended: 2026-09-28T20:28:40Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9312256-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9312256-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7228/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104647187924258816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7228,"pr_state":"closed","pr_sha":"ec4eb40ff1932b90cc19c376b1401d4194dba6a8","final_merged_sha":"ec4eb40ff1932b90cc19c376b1401d4194dba6a8","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T18:59:24Z","started_at":"2026-09-28T18:59:24Z","completed_at":"2026-09-28T21:10:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T21:22:09.259633267Z","timing_metadata_checked_at":"2026-09-28T21:22:09.259633267Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T21:22:09.263137555Z","region_metadata_checked_at":"2026-09-28T21:22:09.263137555Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104641856926126080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7141,"pr_state":"closed","pr_sha":"3300048d8df3d9e2a9338171e944bb89dcc9e6f2","final_merged_sha":"6afba8239efe6245441f25e86ccd2ab7ba7f5035","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T18:38:15Z","started_at":"2026-09-28T18:38:15Z","completed_at":"2026-09-28T21:00:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T21:11:34.498999986Z","timing_metadata_checked_at":"2026-09-28T21:11:34.498999986Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T21:11:34.502417075Z","region_metadata_checked_at":"2026-09-28T21:11:34.502417075Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b123b5d3759f876727c92e50c3f2e0a7cc2e90dd6ee6fe529d5d799581df3299","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104641856926126080","occurred_at":"2026-09-28T18:38:15Z","failed_at":"e2e","signature_id":"da523c4bd417aae8b3768777b0207e474c00bde3f1c27acfd3181f0c1268a499","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001512f00\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-5f68mkqs6vtp\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-5f68mkqs6vtp, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001512f00\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-5f68mkqs6vtp\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-5f68mkqs6vtp, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-542tb2cfkqw8\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-542tb2cfkqw8, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"a6c8934ce1cf8b8783cd8255e13e620a84150dfb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T18:27:03Z","started_at":"2026-09-28T18:27:03Z","completed_at":"2026-09-28T20:43:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T20:50:41.883102089Z","timing_metadata_checked_at":"2026-09-28T20:50:41.883102089Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T20:50:41.886242779Z","region_metadata_checked_at":"2026-09-28T20:50:41.886242779Z"},"failed_at":["alert","e2e"],"failed_test_count":20,"occurrences":[{"row_id":"19c763ef319baa327be14c79409ca416948394e80d2f8fd33f722ad2bf36c72f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"alert","signature_id":"628ec6c53961c8770e007de15e33a4677fa2c411d6198e11b35c424d814ac61c","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 79 time(s) (63 unknown, 16 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T19:22:59Z\n Ended: 2026-09-28T19:45:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"router-5b97c8c479-td767\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/router-5b97c8c479-td767 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T19:23:58Z\n Ended: 2026-09-28T19:45:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rbkaemrkauofk92ubess3a0bmc8a-v20261001\", pod=\"router-688fdf9d74-7b7qb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46rbkaemrkauofk92ubess3a0bmc8a-v20261001/router-688fdf9d74-7b7qb has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-28T19:25:04Z\n Ended: 2026-09-28T19:47:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46s9suqafp0kib2eubo5b5ohtc2om9-agg-cluster\", pod=\"router-7856497d5-p9s72\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46s9suqafp0kib2eubo5b5ohtc2om9-agg-cluster/router-7856497d5-p9s72 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-28T19:25:04Z\n Ended: 2026-09-28T19:47:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46s9suqafp0kib2eubo5b5ohtc2om9-agg-cluster\", pod=\"router-7856497d5-ptl86\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46s9suqafp0kib2eubo5b5ohtc2om9-agg-cluster/router-7856497d5-ptl86 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-28T19:25:59Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"ovnkube-control-plane-5fd96b4979-gbmgt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/ovnkube-control-plane-5fd96b4979-gbmgt has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-28T19:25:59Z\n Ended: 2026-09-28T19:46:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"ingress-operator-7d88f7b5dc-drxfd\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/ingress-operator-7d88f7b5dc-drxfd has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-28T19:25:59Z\n Ended: 2026-09-28T19:33:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e\", pod=\"azure-file-csi-driver-controller-854b478df8-v98lf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e/azure-file-csi-driver-controller-854b478df8-v98lf has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-28T19:25:59Z\n Ended: 2026-09-28T19:32:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e\", pod=\"azure-disk-csi-driver-controller-5785d6645d-6xw4f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e/azure-disk-csi-driver-controller-5785d6645d-6xw4f has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-28T19:25:59Z\n Ended: 2026-09-28T19:32:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e\", pod=\"azure-disk-csi-driver-controller-5c84fd46c4-mpzts\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e/azure-disk-csi-driver-controller-5c84fd46c4-mpzts has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-28T19:25:59Z\n Ended: 2026-09-28T19:33:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e\", pod=\"azure-file-csi-driver-controller-559d66d9f8-v468g\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46re8cgt25tnc3e3r05ha4ola9p2m6-n3q8w5v1a0v4c7e/azure-file-csi-driver-controller-559d66d9f8-v468g has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-28T19:25:59Z\n Ended: 2026-09-28T19:33:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"azure-file-csi-driver-controller-9688c9fcd-vkkwc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/azure-file-csi-driver-controller-9688c9fcd-vkkwc has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-28T19:25:59Z\n Ended: 2026-09-28T19:32:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"azure-disk-csi-driver-controller-df8985bcf-xr5pq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/azure-disk-csi-driver-controller-df8985bcf-xr5pq has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-28T19:25:59Z\n Ended: 2026-09-28T19:32:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"azure-file-csi-driver-controller-6f4b9c57cd-gpkcq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/azure-file-csi-driver-controller-6f4b9c57cd-gpkcq has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-28T19:29:02Z\n Ended: 2026-09-28T19:35:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6qhsdqg59jb18mtn75mdepd994t-p1e0l2c6g7e7x9p\", pod=\"azure-disk-csi-driver-controller-d5b4b46ff-swrjk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6qhsdqg59jb18mtn75mdepd994t-p1e0l2c6g7e7x9p/azure-disk-csi-driver-controller-d5b4b46ff-swrjk has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-28T19:29:02Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"ignition-server-6cf4c4cdbb-dcl2n\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/ignition-server-6cf4c4cdbb-dcl2n has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-28T19:30:02Z\n Ended: 2026-09-28T19:36:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"cluster-storage-operator-665d6949b6-6fw7x\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/cluster-storage-operator-665d6949b6-6fw7x has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-28T19:30:02Z\n Ended: 2026-09-28T19:37:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"ingress-operator-fb65c85-29jmc\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/ingress-operator-fb65c85-29jmc has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-28T19:30:02Z\n Ended: 2026-09-28T19:36:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"dns-operator-bfc958595-f2lvq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/dns-operator-bfc958595-f2lvq has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-28T19:30:02Z\n Ended: 2026-09-28T19:45:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a\", pod=\"kube-controller-manager-847bbfc79-gdtt6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a/kube-controller-manager-847bbfc79-gdtt6 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-28T19:30:02Z\n Ended: 2026-09-28T19:36:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"cluster-network-operator-6995f687db-6cfj4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/cluster-network-operator-6995f687db-6cfj4 has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-28T19:33:01Z\n Ended: 2026-09-28T19:39:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rb6d2v8ktanhkbba2l3aora65d7u-t1b3k0j9k6s4g3e\", pod=\"kube-apiserver-5f69cc7c69-66mrs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46rb6d2v8ktanhkbba2l3aora65d7u-t1b3k0j9k6s4g3e/kube-apiserver-5f69cc7c69-66mrs has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:44:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"cluster-network-operator-79b745b7-2lm4f\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-network-operator-79b745b7-2lm4f has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:46:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create\", pod=\"kube-apiserver-c7844b576-98nm5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create/kube-apiserver-c7844b576-98nm5 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"cluster-version-operator-767f75f6bd-czn9v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-version-operator-767f75f6bd-czn9v has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:45:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"network-node-identity-688597d98d-5nwfj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/network-node-identity-688597d98d-5nwfj has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:46:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i\", pod=\"kube-apiserver-674ffb8fcf-w2vnq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i/kube-apiserver-674ffb8fcf-w2vnq has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:39:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"csi-snapshot-controller-78d9d968f6-lktl7\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/csi-snapshot-controller-78d9d968f6-lktl7 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:45:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"kube-apiserver-7f4977d86c-ggtv4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/kube-apiserver-7f4977d86c-ggtv4 has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:44:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"community-operators-catalog-77957f8659-vrvjj\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/community-operators-catalog-77957f8659-vrvjj has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:44:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"multus-admission-controller-6659c5d88f-hgsf6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/multus-admission-controller-6659c5d88f-hgsf6 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:45:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"router-7964b968d8-s9n7t\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/router-7964b968d8-s9n7t has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"ignition-server-6cf4c4cdbb-xqn6v\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/ignition-server-6cf4c4cdbb-xqn6v has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"openshift-apiserver-79bff85c55-rmr6j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/openshift-apiserver-79bff85c55-rmr6j has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:44:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f\", pod=\"openshift-apiserver-f4f5cc6bf-j27d5\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r9oavdebkcu57noiakdj4edk9fc6-c0q0y0c8u3m6c0f/openshift-apiserver-f4f5cc6bf-j27d5 has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:44:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r619d7598ajep6f8mtkq1cmmun0r-g6f2d0d9e8y9l8w\", pod=\"openshift-apiserver-855bf867d4-5dj69\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r619d7598ajep6f8mtkq1cmmun0r-g6f2d0d9e8y9l8w/openshift-apiserver-855bf867d4-5dj69 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-09-28T19:34:01Z\n Ended: 2026-09-28T19:45:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i\", pod=\"openshift-apiserver-548d95f77-j9rt4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i/openshift-apiserver-548d95f77-j9rt4 has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:46:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"kube-apiserver-8488cbccb8-9cntq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/kube-apiserver-8488cbccb8-9cntq has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:44:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a\", pod=\"openshift-apiserver-59cf74f469-2pkbk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a/openshift-apiserver-59cf74f469-2pkbk has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:44:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"cluster-image-registry-operator-5f9967bf67-m75d8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-image-registry-operator-5f9967bf67-m75d8 has been in a non-ready state for longer than 5 minutes.\nFiring 40:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a\", pod=\"router-6966cb469f-jdx2g\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a/router-6966cb469f-jdx2g has been in a non-ready state for longer than 5 minutes.\nFiring 41:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:46:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"kube-apiserver-6474ccdcd7-j5pkn\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/kube-apiserver-6474ccdcd7-j5pkn has been in a non-ready state for longer than 5 minutes.\nFiring 42:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:45:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a\", pod=\"kube-apiserver-7877d486f4-dwd29\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46raguojej3hq4vd3uacec64tpmrts-g2q4v5m1s0e2c1a/kube-apiserver-7877d486f4-dwd29 has been in a non-ready state for longer than 5 minutes.\nFiring 43:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:45:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"network-node-identity-79f47f8db7-c6lsb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/network-node-identity-79f47f8db7-c6lsb has been in a non-ready state for longer than 5 minutes.\nFiring 44:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:45:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i\", pod=\"network-node-identity-6bf46b86d-r6lxm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r9cvp37q0olg8c158c363ap9mcq5-u6o0x8k8g4t9c8i/network-node-identity-6bf46b86d-r6lxm has been in a non-ready state for longer than 5 minutes.\nFiring 45:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:48:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"router-d5fd8c684-npjp8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/router-d5fd8c684-npjp8 has been in a non-ready state for longer than 5 minutes.\nFiring 46:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:45:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"openshift-apiserver-7fccc5c996-2fqjh\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/openshift-apiserver-7fccc5c996-2fqjh has been in a non-ready state for longer than 5 minutes.\nFiring 47:\n State: Resolved\n Started: 2026-09-28T19:35:03Z\n Ended: 2026-09-28T19:44:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"certified-operators-catalog-56fcb687d5-x55km\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/certified-operators-catalog-56fcb687d5-x55km has been in a non-ready state for longer than 5 minutes.\nFiring 48:\n State: Resolved\n Started: 2026-09-28T19:35:04Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create\", pod=\"openshift-apiserver-58689dc575-d9psz\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create/openshift-apiserver-58689dc575-d9psz has been in a non-ready state for longer than 5 minutes.\nFiring 49:\n State: Resolved\n Started: 2026-09-28T19:35:04Z\n Ended: 2026-09-28T19:45:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"openshift-apiserver-fb96d7cd5-n8vx8\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/openshift-apiserver-fb96d7cd5-n8vx8 has been in a non-ready state for longer than 5 minutes.\nFiring 50:\n State: Resolved\n Started: 2026-09-28T19:35:04Z\n Ended: 2026-09-28T19:45:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"kube-apiserver-58866c8f4c-rssbm\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/kube-apiserver-58866c8f4c-rssbm has been in a non-ready state for longer than 5 minutes.\nFiring 51:\n State: Resolved\n Started: 2026-09-28T19:35:04Z\n Ended: 2026-09-28T19:45:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"openshift-apiserver-57597ccd79-kdsdf\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/openshift-apiserver-57597ccd79-kdsdf has been in a non-ready state for longer than 5 minutes.\nFiring 52:\n State: Resolved\n Started: 2026-09-28T19:35:04Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l\", pod=\"network-node-identity-6b45dfb8b5-gpsw6\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3p7siuaq0ddmvgvscsi8di7b2ms-g5j1a0y1m9t0v0l/network-node-identity-6b45dfb8b5-gpsw6 has been in a non-ready state for longer than 5 minutes.\nFiring 53:\n State: Resolved\n Started: 2026-09-28T19:35:04Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"router-5b97c8c479-87d9j\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/router-5b97c8c479-87d9j has been in a non-ready state for longer than 5 minutes.\nFiring 54:\n State: Resolved\n Started: 2026-09-28T19:35:04Z\n Ended: 2026-09-28T19:45:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create\", pod=\"network-node-identity-cf9fb8b88-4lqsp\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46rdha411s2s213ofkbr2v7t07eh2d-acr-pull-create/network-node-identity-cf9fb8b88-4lqsp has been in a non-ready state for longer than 5 minutes.\nFiring 55:\n State: Resolved\n Started: 2026-09-28T19:35:04Z\n Ended: 2026-09-28T19:45:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"kube-apiserver-6776ddb8b4-jtcfw\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/kube-apiserver-6776ddb8b4-jtcfw has been in a non-ready state for longer than 5 minutes.\nFiring 56:\n State: Resolved\n Started: 2026-09-28T19:36:01Z\n Ended: 2026-09-28T19:44:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"azure-file-csi-driver-controller-68d9cdbd6c-bjqhs\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/azure-file-csi-driver-controller-68d9cdbd6c-bjqhs has been in a non-ready state for longer than 5 minutes.\nFiring 57:\n State: Resolved\n Started: 2026-09-28T19:36:02Z\n Ended: 2026-09-28T19:44:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"azure-disk-csi-driver-controller-688579f7b8-gcwdq\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/azure-disk-csi-driver-controller-688579f7b8-gcwdq has been in a non-ready state for longer than 5 minutes.\nFiring 58:\n State: Resolved\n Started: 2026-09-28T19:36:02Z\n Ended: 2026-09-28T19:44:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"azure-disk-csi-driver-controller-778588d6c6-grn9w\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/azure-disk-csi-driver-controller-778588d6c6-grn9w has been in a non-ready state for longer than 5 minutes.\nFiring 59:\n State: Resolved\n Started: 2026-09-28T19:36:02Z\n Ended: 2026-09-28T19:45:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"kube-controller-manager-5488d69fb6-p9ltk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/kube-controller-manager-5488d69fb6-p9ltk has been in a non-ready state for longer than 5 minutes.\nFiring 60:\n State: Resolved\n Started: 2026-09-28T19:37:01Z\n Ended: 2026-09-28T19:44:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"kube-controller-manager-9864948dd-qzgnt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/kube-controller-manager-9864948dd-qzgnt has been in a non-ready state for longer than 5 minutes.\nFiring 61:\n State: Resolved\n Started: 2026-09-28T19:37:01Z\n Ended: 2026-09-28T19:44:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"ovnkube-control-plane-58c8f9b6c5-45j6m\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/ovnkube-control-plane-58c8f9b6c5-45j6m has been in a non-ready state for longer than 5 minutes.\nFiring 62:\n State: Resolved\n Started: 2026-09-28T19:37:01Z\n Ended: 2026-09-28T19:44:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k\", pod=\"network-node-identity-789b89cf4b-2v5sk\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46r6c80n9jpgk27jtqr1mg2ert1grv-h9i2w9m0c5c0h0k/network-node-identity-789b89cf4b-2v5sk has been in a non-ready state for longer than 5 minutes.\nFiring 63:\n State: Resolved\n Started: 2026-09-28T19:38:02Z\n Ended: 2026-09-28T19:44:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0189952-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t46re3hoaml91vd90m68oiaekbg44as-e7l1h2p9d0o6f4j\", pod=\"openshift-apiserver-5f8cc55fd4-bm995\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t46re3hoaml91vd90m68oiaekbg44as-e7l1h2p9d0o6f4j/openshift-apiserver-5f8cc55fd4-bm995 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"24c904f3848a3ffc53e5099d25403a0251e6b49ad5803ee4f237e2bf6b41570d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"alert","signature_id":"d0d5fd846dafdaf7af7c9317c65a2b6639e493453a72d807fd729028c2663758","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46r598rg9utsesqokqgsh5is4bsff4-private-kas/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46ri3b6cnjabfmbna1k3ic8p591bg1-n6n8b8u3w4n1g4l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-09-28T20:21:24Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/hcps-j0189952\", namespace=\"ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t46r3spp95klv4no3blpdjir2phpkgs-v4e7b2k4r8i1q2z/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"45fe6e02a362a70ee73344de315de1673c3ed7b6e0eb2b23fa824a2a17688baf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"alert","signature_id":"c7cb04f07a33dee0036fa4a43a64560ad19f5cf76e72a8e93d2645cb223f7e32","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T19:37:02Z\n Ended: 2026-09-28T19:45:06Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T19:39:13Z\n Ended: 2026-09-28T19:45:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-28T19:39:13Z\n Ended: 2026-09-28T19:45:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"swift-recorder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"swift-recorder\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet swift-recorder/swift-recorder has not finished or progressed for at least 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-28T19:39:13Z\n Ended: 2026-09-28T19:45:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"472c73b3d12507e15d22c108909959ad8314cea3c220bdab2c40001152be98f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"7b7a706cc0ff1c912898b8d7c4500fb2e9a9ce840a6f74f1d3c202e4b6252f67","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f1a240\u003e: \n resource group \"rg-candidate-4-23-8lfxrmnvvv2w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-8hk4r9\" in resourcegroup=\"rg-candidate-4-23-8lfxrmnvvv2w\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7972cdda-000f-4549-95ba-7b26044f70b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7972cdda-000f-4549-95ba-7b26044f70b4\",\n \"name\": \"7972cdda-000f-4549-95ba-7b26044f70b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:49:57.362020562Z\",\n \"endTime\": \"2026-09-28T20:14:00.292519621Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46r6c80n9jpgk27jtqr1mg2ert1grv still exists (deletion dispatched at 2026-09-28T19:49:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f1a240\u003e: \n resource group \"rg-candidate-4-23-8lfxrmnvvv2w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-8hk4r9\" in resourcegroup=\"rg-candidate-4-23-8lfxrmnvvv2w\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7972cdda-000f-4549-95ba-7b26044f70b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7972cdda-000f-4549-95ba-7b26044f70b4\",\n \"name\": \"7972cdda-000f-4549-95ba-7b26044f70b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:49:57.362020562Z\",\n \"endTime\": \"2026-09-28T20:14:00.292519621Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46r6c80n9jpgk27jtqr1mg2ert1grv still exists (deletion dispatched at 2026-09-28T19:49:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4b0e5d64c55c55acb5f4793c338c7d3e4bd7a517a5711142651af96af4ef08e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"632ad063d78546b35d3640152b34002f194bf542539b83c541017997e134011f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-b6k7qp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000722f40\u003e: \n failed to create route: Internal error occurred: admission plugin \"MutatingAdmissionWebhook\" failed to complete mutation in 13s\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-b6k7qp\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000722f40\u003e: \n failed to create route: Internal error occurred: admission plugin \"MutatingAdmissionWebhook\" failed to complete mutation in 13s\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2458db19d1aa1c830d2005883aecaebf402e5888aa2957a0d46c624d33b2bd43","failure_pattern":"failed to create route: Internal error occurred: admission plugin \"MutatingAdmissionWebhook\" failed to complete mutation in 13s","search_query":"failed to create route: Internal error occurred: admission plugin \"MutatingAdmissionWebhook\" failed to complete mutation in 13s"}},{"row_id":"504b030b70649c175b78f53ab8232cdfdcbd14afe28b9aff14a5fde06d464082","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"0b09d572458a381dac48ffb0eb837c9df3852335997b0c4dc2a876b33b08a4d9","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00120a3a8\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-zgxl8s4cd9df\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6724cdd6-177b-4593-9911-87900b80ceca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6724cdd6-177b-4593-9911-87900b80ceca\",\n \"name\": \"6724cdd6-177b-4593-9911-87900b80ceca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:42:59.042442346Z\",\n \"endTime\": \"2026-09-28T20:02:05.640354291Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2k9vl-krm2l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zgxl8s4cd9df--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2k9vl-krm2l to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2k9vl-krm2l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zgxl8s4cd9df--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2k9vl-krm2l to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc00120a3a8\u003e: \n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-zgxl8s4cd9df\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6724cdd6-177b-4593-9911-87900b80ceca\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/6724cdd6-177b-4593-9911-87900b80ceca\",\n \"name\": \"6724cdd6-177b-4593-9911-87900b80ceca\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:42:59.042442346Z\",\n \"endTime\": \"2026-09-28T20:02:05.640354291Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine agg-cluster-agg-np-a-2k9vl-krm2l: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zgxl8s4cd9df--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2k9vl-krm2l to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-2k9vl-krm2l: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-zgxl8s4cd9df--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-2k9vl-krm2l to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5e8cb2f967a52d819183a3a0a2b84ac7fa4c8659d3e7b372b428c04b4eba0d28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"883e6909996fc32433cd23f6eb465299bc9e4798451dc2382f9ae8bc9046e7f6","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-8fvg54\" with version 4.22.15 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae0680\u003e: \n failed to create HCP cluster cluster-zstream-4-22-8fvg54: failed waiting for cluster=\"cluster-zstream-4-22-8fvg54\" in resourcegroup=\"rg-zstream-upgrade-4-22-8zfwm76xgnlj\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/847235cf-8889-474c-8fc4-418de69191f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/847235cf-8889-474c-8fc4-418de69191f0\",\n \"name\": \"847235cf-8889-474c-8fc4-418de69191f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:18.057920805Z\",\n \"endTime\": \"2026-09-28T19:39:25.918712924Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-8fvg54\" with version 4.22.15 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae0680\u003e: \n failed to create HCP cluster cluster-zstream-4-22-8fvg54: failed waiting for cluster=\"cluster-zstream-4-22-8fvg54\" in resourcegroup=\"rg-zstream-upgrade-4-22-8zfwm76xgnlj\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/847235cf-8889-474c-8fc4-418de69191f0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/847235cf-8889-474c-8fc4-418de69191f0\",\n \"name\": \"847235cf-8889-474c-8fc4-418de69191f0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:18.057920805Z\",\n \"endTime\": \"2026-09-28T19:39:25.918712924Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb1198fd68f10a5126a38488adcab0e5698ddefc9bae064f4179ead558afbe1a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"79d2b0aa81e9711c334e453972205661be0c4d5ecfe68f53043ddbfb1aef4e2d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"be02b062470985c866d3bce0cf574ecc16a3fffda367f6f061cf26ba857913fb","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-27p526\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d20a60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-27p526: failed waiting for cluster=\"cp-ystream-upgrade-4-22-27p526\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7drm72sqb8zr\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\",\n \"name\": \"3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:49.293974095Z\",\n \"endTime\": \"2026-09-28T19:39:55.897408712Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-27p526\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d20a60\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-27p526: failed waiting for cluster=\"cp-ystream-upgrade-4-22-27p526\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-7drm72sqb8zr\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\",\n \"name\": \"3b089068-0a7a-4b3d-bdf3-9f03ebedd6b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:49.293974095Z\",\n \"endTime\": \"2026-09-28T19:39:55.897408712Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb1198fd68f10a5126a38488adcab0e5698ddefc9bae064f4179ead558afbe1a","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7e7482f79918cfb25fb64e0f5a7abffafcb7a9954a5794e703436115b0f6ab67","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"ae7ec42dfdec2a94af8e0db3707b53e37791d04766056ed14bd22a0a1022e2d0","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c241a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-c8jw5nmfkndx\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4538241d-2021-40b7-8228-fb4052b1f808\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4538241d-2021-40b7-8228-fb4052b1f808\",\n \"name\": \"4538241d-2021-40b7-8228-fb4052b1f808\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:37.948094059Z\",\n \"endTime\": \"2026-09-28T19:39:45.568048903Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c241a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-c8jw5nmfkndx\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4538241d-2021-40b7-8228-fb4052b1f808\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4538241d-2021-40b7-8228-fb4052b1f808\",\n \"name\": \"4538241d-2021-40b7-8228-fb4052b1f808\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:37.948094059Z\",\n \"endTime\": \"2026-09-28T19:39:45.568048903Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"825a005ba988ded05aac1767705c0aa2e69d06880e013d882148e88d3bd25f75","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"ea6fad37e59eac223c8a29baca02fe524e66bda14470a0e205eea9919102b3f0","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001400360\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-sl29bx8fv8nz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-sl29bx8fv8nz, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001400360\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-sl29bx8fv8nz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-sl29bx8fv8nz, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-sl29bx8fv8nz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-sl29bx8fv8nz, error: context deadline exceeded"}},{"row_id":"9f80ccf611c093ac7ab3d89b370c314f544a4c520ed943f4e71f1d40ff66986d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"alert","signature_id":"3bdd3d319d232da25d5a6952031f06bb164d2fbc1f8e8e5f1df140b9e37b67e4","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T19:18:56Z\n Ended: 2026-09-28T19:51:55Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0189952-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0189952-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"a06b73154a670060ca35d7e815a54f20d4c5436a85ff92e2a7fa588f3d84f1b2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"fe3ed91dd61bff3602cc82c1188d31cd88e9091fc80ea4f192a5dbe2377f4f68","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000daa858\u003e: \n resource group \"rg-hs-pre--zxrbtldmj8bt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-lz8thl\" in resourcegroup=\"rg-hs-pre--zxrbtldmj8bt\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9657555-53b8-49b9-a717-afcd39c7e1e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9657555-53b8-49b9-a717-afcd39c7e1e0\",\n \"name\": \"a9657555-53b8-49b9-a717-afcd39c7e1e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:30:58.262633809Z\",\n \"endTime\": \"2026-09-28T19:55:05.757251457Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46r69lkt992hqb9p61rfqbkob5804i still exists (deletion dispatched at 2026-09-28T19:30:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000daa858\u003e: \n resource group \"rg-hs-pre--zxrbtldmj8bt\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-hs-pre-lz8thl\" in resourcegroup=\"rg-hs-pre--zxrbtldmj8bt\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9657555-53b8-49b9-a717-afcd39c7e1e0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a9657555-53b8-49b9-a717-afcd39c7e1e0\",\n \"name\": \"a9657555-53b8-49b9-a717-afcd39c7e1e0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:30:58.262633809Z\",\n \"endTime\": \"2026-09-28T19:55:05.757251457Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46r69lkt992hqb9p61rfqbkob5804i still exists (deletion dispatched at 2026-09-28T19:30:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a3cbb95838588bd9331f3fb56fc3fb40df15560ab277cd22575886392b7e01d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"0e99260da9e615074014e3173244136846aafa91e1232cd4f863676446c4f26c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-6mht66l8hhvp/cluster-candidate-4-22-hxvn25 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070f360\u003e: \n failed to create HCP cluster cluster-candidate-4-22-hxvn25: failed waiting for cluster=\"cluster-candidate-4-22-hxvn25\" in resourcegroup=\"rg-candidate-4-22-6mht66l8hhvp\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/889a73a1-f8c7-4ed8-aad9-e3dfdb686168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/889a73a1-f8c7-4ed8-aad9-e3dfdb686168\",\n \"name\": \"889a73a1-f8c7-4ed8-aad9-e3dfdb686168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:32.906905274Z\",\n \"endTime\": \"2026-09-28T19:39:35.528660951Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m1s ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-6mht66l8hhvp/cluster-candidate-4-22-hxvn25 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00070f360\u003e: \n failed to create HCP cluster cluster-candidate-4-22-hxvn25: failed waiting for cluster=\"cluster-candidate-4-22-hxvn25\" in resourcegroup=\"rg-candidate-4-22-6mht66l8hhvp\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/889a73a1-f8c7-4ed8-aad9-e3dfdb686168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/889a73a1-f8c7-4ed8-aad9-e3dfdb686168\",\n \"name\": \"889a73a1-f8c7-4ed8-aad9-e3dfdb686168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:32.906905274Z\",\n \"endTime\": \"2026-09-28T19:39:35.528660951Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 23m1s ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, kube-controller-manager deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c7fe0af289e2969c7eb39488d64d294fadea2123920704d81e8b66750b4f367b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, kube-controller-manager, openshift-apiserver, packageserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a44caf6a01f70389a0e1e0a1d1be0afa806228faf9543e1af678ac0dbe81241e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"6fe4cd9eafd43e0c6bf005c09b6f95529ba4135a257448638db3146d0a4fbe8a","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002154570\u003e: \n resource group \"complex-cilium-kv-jndv4cm2shzf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jndv4cm2shzf\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e40876ab-da07-40cc-bca8-2d36902295a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e40876ab-da07-40cc-bca8-2d36902295a7\",\n \"name\": \"e40876ab-da07-40cc-bca8-2d36902295a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:31:44.581294657Z\",\n \"endTime\": \"2026-09-28T19:55:45.796890355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46rtrm9n8ddf2ubgd68c1s64e1ioap still exists (deletion dispatched at 2026-09-28T19:31:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002154570\u003e: \n resource group \"complex-cilium-kv-jndv4cm2shzf\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-jndv4cm2shzf\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e40876ab-da07-40cc-bca8-2d36902295a7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e40876ab-da07-40cc-bca8-2d36902295a7\",\n \"name\": \"e40876ab-da07-40cc-bca8-2d36902295a7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:31:44.581294657Z\",\n \"endTime\": \"2026-09-28T19:55:45.796890355Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46rtrm9n8ddf2ubgd68c1s64e1ioap still exists (deletion dispatched at 2026-09-28T19:31:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"af3ab0a07f10f15bbce221ed97403c9e810026d5ce92dd506bcd0e1980c0459c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"alert","signature_id":"34ad39f09dd40559811aad8fffcaf6d1a0c0923be2148f882fd171659b3c1d97","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T19:45:04Z\n Ended: 2026-09-28T19:51:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0189952-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.44:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0189952/providers/microsoft.monitor/accounts/services-j0189952\", namespace=\"klusterlet-2t46re8cgt25tnc3e3r05ha4ola9p2m6\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-xp2wg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t46re8cgt25tnc3e3r05ha4ola9p2m6/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"c559d44e47e8cd03082c4dac1fee537c6b6793b20a259e2989ad0f9b52a49cea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"781e63180597c1b1b938f67cc9e5dd0218e1b6f57f1eae6c6e300c0f327db76e","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0013e4180\u003e: \n resource group \"private-keyvault-bzhkhkm6vpw7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-bzhkhkm6vpw7\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/486f914e-d8e9-4103-988d-8630bbe515a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/486f914e-d8e9-4103-988d-8630bbe515a1\",\n \"name\": \"486f914e-d8e9-4103-988d-8630bbe515a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:29:30.639536976Z\",\n \"endTime\": \"2026-09-28T19:53:35.789612068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46rt8qnc76ptkosa2u3fv39hps3u1f still exists (deletion dispatched at 2026-09-28T19:29:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0013e4180\u003e: \n resource group \"private-keyvault-bzhkhkm6vpw7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-bzhkhkm6vpw7\" to finish deleting: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/486f914e-d8e9-4103-988d-8630bbe515a1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/486f914e-d8e9-4103-988d-8630bbe515a1\",\n \"name\": \"486f914e-d8e9-4103-988d-8630bbe515a1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:29:30.639536976Z\",\n \"endTime\": \"2026-09-28T19:53:35.789612068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t46rt8qnc76ptkosa2u3fv39hps3u1f still exists (deletion dispatched at 2026-09-28T19:29:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c68863d0556f5dd0ffb219159231e98bd997704a329cf5861fdf6101e91894c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"65f361db2ca8a207b71ef63dfd951eb9adaf0933223cf15a0e8cb91c8b77bb7d","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060c440\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-br7vgttbd9zx\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\",\n \"name\": \"15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:44.986989982Z\",\n \"endTime\": \"2026-09-28T19:39:45.910063992Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 24m39s ago; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00060c440\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-br7vgttbd9zx\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\",\n \"name\": \"15a2a59b-feef-488b-a5fa-5bcdbdb54fbf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:44.986989982Z\",\n \"endTime\": \"2026-09-28T19:39:45.910063992Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 24m39s ago; hosted cluster degraded: UnavailableReplicas: kube-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e3e3516e4f9f9dcb22ed9c70498bb550fb681b3348ceed9a50619cee4e93ae7","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [kube-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c8b968604982a6e4e2bdbdf2266fd909abe02e241bff5ba1ba70cfcc1d2b72d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"1aeed5a6216935410be4afa173f9b01087ed1978311df09a7db6cc003d86fa29","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-4rl6n98blqls/cluster-candidate-5-0-9zqn4p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f22920\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9zqn4p: failed waiting for cluster=\"cluster-candidate-5-0-9zqn4p\" in resourcegroup=\"rg-candidate-5-0-4rl6n98blqls\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b919a89b-ddbd-409e-ae0b-825760357927\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b919a89b-ddbd-409e-ae0b-825760357927\",\n \"name\": \"b919a89b-ddbd-409e-ae0b-825760357927\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:07.120894243Z\",\n \"endTime\": \"2026-09-28T19:39:16.873419842Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-4rl6n98blqls/cluster-candidate-5-0-9zqn4p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f22920\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9zqn4p: failed waiting for cluster=\"cluster-candidate-5-0-9zqn4p\" in resourcegroup=\"rg-candidate-5-0-4rl6n98blqls\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b919a89b-ddbd-409e-ae0b-825760357927\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b919a89b-ddbd-409e-ae0b-825760357927\",\n \"name\": \"b919a89b-ddbd-409e-ae0b-825760357927\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:07.120894243Z\",\n \"endTime\": \"2026-09-28T19:39:16.873419842Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ad05e6869aca7a4924e8d188ef3355a20d9fcfa6fdf28bdb0dceaf7eb87d2816","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, openshift-apiserver, packageserver, redhat-operators-catalog, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e35c58e2e98493d6bef3c91020f3b126b5ba9d76c1bcc1dd6e354fdb6194bdbc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"8b5a7eabb5b08cdfee8da155099ec3786439ab3265705a61822bd89213b242a9","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010432a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-66ncz7vqwm2z\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35d648d-395d-429f-9f2d-24eed4314d86\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35d648d-395d-429f-9f2d-24eed4314d86\",\n \"name\": \"c35d648d-395d-429f-9f2d-24eed4314d86\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:11.81451702Z\",\n \"endTime\": \"2026-09-28T19:39:16.076223121Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:99]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010432a0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-66ncz7vqwm2z\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35d648d-395d-429f-9f2d-24eed4314d86\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c35d648d-395d-429f-9f2d-24eed4314d86\",\n \"name\": \"c35d648d-395d-429f-9f2d-24eed4314d86\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:10:11.81451702Z\",\n \"endTime\": \"2026-09-28T19:39:16.076223121Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, community-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"29d7e57f52a348cb16f8150b148a3d89306bfe96e3707f80643c1a84b22eed32","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, community-operators-catalog, kube-apiserver, openshift-apiserver, packageserver, redhat-operators-catalog, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f8c54658b979376199fdde6ce18a6bdb42dde4b74f1835b874276d566ec6a911","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104639037800189952","occurred_at":"2026-09-28T18:27:03Z","failed_at":"e2e","signature_id":"0b5f642a5234bd0a643829d4d6a02a3ececb750e0fc106623478f6d9c53767e1","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bf61a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lhz84lf5nw65\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/045ddd50-d43d-42a0-bcda-90888c378119\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/045ddd50-d43d-42a0-bcda-90888c378119\",\n \"name\": \"045ddd50-d43d-42a0-bcda-90888c378119\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:11:03.945176993Z\",\n \"endTime\": \"2026-09-28T19:40:05.721943753Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m29s ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bf61a0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-lhz84lf5nw65\" to finish creating: GET https://rp.j0189952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/045ddd50-d43d-42a0-bcda-90888c378119\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/045ddd50-d43d-42a0-bcda-90888c378119\",\n \"name\": \"045ddd50-d43d-42a0-bcda-90888c378119\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T19:11:03.945176993Z\",\n \"endTime\": \"2026-09-28T19:40:05.721943753Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m29s ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, redhat-marketplace-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d71a4287cac7c4db8dd3797eb946a1bbbe45de0ccd7e568821596e513bc09abb","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, community-operators-catalog, kube-apiserver, redhat-marketplace-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":20,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","2458db19d1aa1c830d2005883aecaebf402e5888aa2957a0d46c624d33b2bd43","29d7e57f52a348cb16f8150b148a3d89306bfe96e3707f80643c1a84b22eed32","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","8e3e3516e4f9f9dcb22ed9c70498bb550fb681b3348ceed9a50619cee4e93ae7","ad05e6869aca7a4924e8d188ef3355a20d9fcfa6fdf28bdb0dceaf7eb87d2816","c7fe0af289e2969c7eb39488d64d294fadea2123920704d81e8b66750b4f367b","cb1198fd68f10a5126a38488adcab0e5698ddefc9bae064f4179ead558afbe1a","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","d71a4287cac7c4db8dd3797eb946a1bbbe45de0ccd7e568821596e513bc09abb","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":20,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104638998440841216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"a6c8934ce1cf8b8783cd8255e13e620a84150dfb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T18:26:54Z","started_at":"2026-09-28T18:26:54Z","completed_at":"2026-09-28T18:27:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T19:14:28.692197239Z","timing_metadata_checked_at":"2026-09-28T19:14:28.692197239Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-28T19:14:28.695111529Z","region_metadata_checked_at":"2026-09-28T20:19:07.443536118Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0f1ce7cf29cc551b54f95a95771ccc46aa1e4bb03997e7b99f6706a629884d1e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104638998440841216","occurred_at":"2026-09-28T18:26:54Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104634571122282496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"459a7d15c239bb94793dd5eead8140e6a04323f5","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T18:08:52Z","started_at":"2026-09-28T18:08:52Z","completed_at":"2026-09-28T20:25:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T20:40:24.329707098Z","timing_metadata_checked_at":"2026-09-28T20:40:24.329707098Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T20:40:24.333551386Z","region_metadata_checked_at":"2026-09-28T20:40:24.333551386Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"16c2936b4ea8af3d8bd11ac92eb8fe0dfa9bbe9f98ad07f341d57db50292b121","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104634571122282496","occurred_at":"2026-09-28T18:08:52Z","failed_at":"e2e","signature_id":"59553741fb4c75e51213f6e718afb7cc45e4ae5d08010e66f0c43031bbb2a5f9","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0xc002737518\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:251]: failed to verify cluster operators are available after external auth config creation\nUnexpected error:\n \u003c*errors.joinError | 0xc002737518\u003e: \n context deadline exceeded\n failed to list ClusterOperators: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"8fa63251e5889035fe3c0d99fdea121b60a027a8b1f9488c7713bc1a3224adef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104634571122282496","occurred_at":"2026-09-28T18:08:52Z","failed_at":"e2e","signature_id":"4f3c97c58357c28ebf837088dc673173d0f261c2fec843f5f7f22dbbe9566892","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:243]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullnsjbs8pwqtmk.azurecr.io\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101eb60\u003e: \n VerifyImagePulled(acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal) timed out after 10m0.008s: expected image from acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal in namespace acr-pull-test-day2: image pull errors:\n - pod acr-pull-test container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: e8a1ddc1-f3e1-4c99-ae14-bbe272e0b702; artifact err: image reference: get manifest from ref: create image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 7d311f24-e917-4750-a06e-94b359669923\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:243]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullnsjbs8pwqtmk.azurecr.io\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101eb60\u003e: \n VerifyImagePulled(acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal) timed out after 10m0.008s: expected image from acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal in namespace acr-pull-test-day2: image pull errors:\n - pod acr-pull-test container acr-pull-test: ImagePullBackOff - Back-off pulling image \"acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal:latest\": ErrImagePull: unable to pull image or OCI artifact: pull image err: initializing source docker://acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal:latest: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: e8a1ddc1-f3e1-4c99-ae14-bbe272e0b702; artifact err: image reference: get manifest from ref: create image source: unable to retrieve auth token: invalid username/password: unauthorized: authentication required, visit https://aka.ms/acr/authorization for more information. CorrelationId: 7d311f24-e917-4750-a06e-94b359669923\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"81045ed261487eb0846f85a3e15e6408c3110ed61e79109821b9fdbaebb04f4d","failure_pattern":"VerifyImagePulled(acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal) timed out after 10m0.008s: expected image from acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal in namespace acr-pull-test-day2: image pull errors:","search_query":"VerifyImagePulled(acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal) timed out after 10m0.008s: expected image from acrpullnsjbs8pwqtmk.azurecr.io/ubi-minimal in namespace acr-pull-test-day2: image pull errors:"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","81045ed261487eb0846f85a3e15e6408c3110ed61e79109821b9fdbaebb04f4d"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104626712733028352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7053,"pr_state":"closed","pr_sha":"3e077fa473e1d0fb9f27fc5472f089cc2bc994d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T17:38:04Z","started_at":"2026-09-28T17:38:04Z","completed_at":"2026-09-28T19:46:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T19:57:27.99902237Z","timing_metadata_checked_at":"2026-09-28T19:57:27.99902237Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T19:57:28.002656859Z","region_metadata_checked_at":"2026-09-28T19:57:28.002656859Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"282a618faa57a06f9460ac6028156b5de1e0ee596850967cd4a11757e1dc8eee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104626712733028352","occurred_at":"2026-09-28T17:38:04Z","failed_at":"alert","signature_id":"ad745131b08863f15bc2d4da07c88878d389ce2e6a6b3cb61447530799f60d59","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T18:59:41Z\n Ended: 2026-09-28T19:05:43Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3028352-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3028352-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104623441356263424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"open","pr_sha":"f51a6f979cdcd8fbe30de8353ac9efec7fe08656","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T17:25:03Z","started_at":"2026-09-28T17:25:03Z","completed_at":"2026-09-28T19:40:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T19:46:56.05608496Z","timing_metadata_checked_at":"2026-09-28T19:46:56.05608496Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T19:46:56.059708148Z","region_metadata_checked_at":"2026-09-28T19:46:56.059708148Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"027383c439940701bee6b3ee56b376f5bb2655344bbbe09580c1c9b459675aad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104623441356263424","occurred_at":"2026-09-28T17:25:03Z","failed_at":"alert","signature_id":"874dec276004f85c33608d553f88244c23c135754a3f5df97b8525a573d31443","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T18:17:15Z\n Ended: 2026-09-28T18:51:14Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6263424-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6263424-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104620444073070592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7227,"pr_state":"open","pr_sha":"a3afefb47d8a3b2dc3f8e556d6617d4384971aa7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T17:13:10Z","started_at":"2026-09-28T17:13:10Z","completed_at":"2026-09-28T19:30:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T19:36:05.199257526Z","timing_metadata_checked_at":"2026-09-28T19:36:05.199257526Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T19:36:05.204676408Z","region_metadata_checked_at":"2026-09-28T19:36:05.204676408Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b53b256fe7cb45a6943e90aaff49cee35d72e89961efb2134789eeae1ba00ed0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7227/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104620444073070592","occurred_at":"2026-09-28T17:13:10Z","failed_at":"alert","signature_id":"ac6a9a0ba19953408fe70e4eaa33691a22600749799a03e7a8a0d5f34b42d29b","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T18:06:05Z\n Ended: 2026-09-28T18:40:06Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3070592-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3070592-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104618768113078272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"629f088f62b6a620bbeec1cd7b6590d91cf6c63a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T17:06:26Z","started_at":"2026-09-28T17:06:26Z","completed_at":"2026-09-28T17:54:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:02:00.686970143Z","timing_metadata_checked_at":"2026-09-28T18:02:00.686970143Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:02:00.689567835Z","region_metadata_checked_at":"2026-09-28T18:02:00.689567835Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"7bb3d968b66be559bab9984269df4afb0fbd0f4ae6c376a3b0897b06883a2e6b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104618768113078272","occurred_at":"2026-09-28T17:06:26Z","failed_at":"provision","signature_id":"3cf8fbf66e63c9add929754f6f22e240822d57f521f0d283a39c4b61044dd3ef","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T17:38:03.261Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T17:45:40.947Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T17:38:03.27081091Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T17:38:05.230722026Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T17:38:05.332101871Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944777574Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"FDC79503A5389868\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T17:43:12.5777514Z\\\",\\\"duration\\\":\\\"PT1M36.8422604S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"4\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41110440-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 2, 1), 'Standard_E8ds_v6' (zones 1, 2), 'Standard_E8-4ds_v6' (zones 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41110440-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944878586Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"DC248B23802810EC\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:43:12.5777514Z\\\",\\\"duration\\\":\\\"PT1M22.2984424S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944892486Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"F64F2B13FFA5C10A\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:43:12.5777514Z\\\",\\\"duration\\\":\\\"PT2M10.1227436S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944904226Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"061A995FDE246942\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:38:09.8797046Z\\\",\\\"duration\\\":\\\"PT6M25.5728844S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944914146Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"4E79D1B051092E6E\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:43:12.5777514Z\\\",\\\"duration\\\":\\\"PT1M21.7193107S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944924847Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E245F41D17E48B12\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T17:38:09.8797046Z\\\",\\\"duration\\\":\\\"PT5M2.6044727S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T17:45:40.944945037Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3078272-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j3078272-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109897999722695\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j3078272-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41110440-vmss'. Please try alternative sizes 'Standard_D8s_v6' (zones 2, 1), 'Standard_E8ds_v6' (zones 1, 2), 'Standard_E8-4ds_v6' (zones 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3078272-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-41110440-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"4\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m37.674139897s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7124,"pr_state":"closed","pr_sha":"fd6568fd4192c8c0b99300449451b369f91be9db","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T16:45:41Z","started_at":"2026-09-28T16:45:41Z","completed_at":"2026-09-28T18:41:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:53:18.533588207Z","timing_metadata_checked_at":"2026-09-28T18:53:18.533588207Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:53:18.539729787Z","region_metadata_checked_at":"2026-09-28T18:53:18.539729787Z"},"failed_at":["e2e"],"failed_test_count":8,"occurrences":[{"row_id":"0c07d514ecb7a61ca221bbee7defbd77f8cbf43b7f721890985b1c596d7e1a76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","failed_at":"e2e","signature_id":"af77cab624dfbc834f82810d101f0f6241da610632987ef8d562e3944af01ee4","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb0020\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb0020\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b2ed02918fbe605f5650b4b12180f78e9cff28da794fad98a9e5cfcbe1ec8f13","failure_pattern":"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.005s: DaemonSet \u003cnamespace\u003e/global-pull-secret-syncer has no desired pods scheduled","search_query":"occurred"}},{"row_id":"3e33f651de0ab73912134d9b4ef77116489f57216ac6813deb8bfd2cfb8150aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","failed_at":"e2e","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:755]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade"}},{"row_id":"62155753f443c467b63cd01f80cb796be88480d4733907d2315866f1eb293225","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","failed_at":"e2e","signature_id":"c178e32f3fda3138a135eb9850e356b70cd5c3cfe3691505d0aae5fb9b47aa1b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wsvb2z\" for cluster \"cluster-candidate-4-20-wsvb2z\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f712c0\u003e: \n failed to create NodePool np-wsvb2z: failed waiting for nodepool=\"np-wsvb2z\" for cluster \"cluster-candidate-4-20-wsvb2z\" in resourcegroup=\"rg-candidate-4-20-slrjsf69bdm7\" to finish creating: GET https://rp.j7194880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/119c8117-1a73-44d6-8ce8-38b6990193b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/119c8117-1a73-44d6-8ce8-38b6990193b7\",\n \"name\": \"119c8117-1a73-44d6-8ce8-38b6990193b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T17:43:51.079865069Z\",\n \"endTime\": \"2026-09-28T18:02:59.523800413Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-wsvb2z\" for cluster \"cluster-candidate-4-20-wsvb2z\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f712c0\u003e: \n failed to create NodePool np-wsvb2z: failed waiting for nodepool=\"np-wsvb2z\" for cluster \"cluster-candidate-4-20-wsvb2z\" in resourcegroup=\"rg-candidate-4-20-slrjsf69bdm7\" to finish creating: GET https://rp.j7194880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/119c8117-1a73-44d6-8ce8-38b6990193b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/119c8117-1a73-44d6-8ce8-38b6990193b7\",\n \"name\": \"119c8117-1a73-44d6-8ce8-38b6990193b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T17:43:51.079865069Z\",\n \"endTime\": \"2026-09-28T18:02:59.523800413Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"90f091c63f3a92fd77be1b2b13256e1deb2c0c1c60e7136a9cfce99541cc3186","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:476]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:476]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"9af418a492ce5ce1438c542846d5d332afab3c813efe2451edad1e0f0417d8c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","failed_at":"e2e","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade"}},{"row_id":"b81ab2bacd06d172b98206ddc612e7b5361414e7c5361d11bd772e8af137612e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","failed_at":"e2e","signature_id":"e69c1d42b103c2fc2619e82648d81646d2d6df78f72c3c1d170576fb995e0bdb","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb6980\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-v4wmmc9xsjpk\" to finish creating: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bb6980\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-v4wmmc9xsjpk\" to finish creating: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004","failure_pattern":"failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context deadline exceeded","search_query":"failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-v4wmmc9xsjpk\" to finish creating: context deadline exceeded"}},{"row_id":"bd866e6b7f764b0b6e3e81dd22d6ea32b78e4a2793151ed4114ac4cf7d9fe890","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","failed_at":"e2e","signature_id":"69e6cd1f2acc497163a0ca5d4d17a7b4e30d385be065f675b68acee156ec0d15","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce94a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-j9cxlp\" in resourcegroup=\"rg-np-noedge-j9cxlp-9zvfnm962trj\" to finish creating: GET https://rp.j7194880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c1e081e-0b8b-4062-9367-d3e3f4309da6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c1e081e-0b8b-4062-9367-d3e3f4309da6\",\n \"name\": \"3c1e081e-0b8b-4062-9367-d3e3f4309da6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T17:41:00.477362299Z\",\n \"endTime\": \"2026-09-28T18:00:09.600349173Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ce94a0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-j9cxlp\" in resourcegroup=\"rg-np-noedge-j9cxlp-9zvfnm962trj\" to finish creating: GET https://rp.j7194880.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c1e081e-0b8b-4062-9367-d3e3f4309da6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3c1e081e-0b8b-4062-9367-d3e3f4309da6\",\n \"name\": \"3c1e081e-0b8b-4062-9367-d3e3f4309da6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T17:41:00.477362299Z\",\n \"endTime\": \"2026-09-28T18:00:09.600349173Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: No Machines are created; node pool condition AllMachinesReady is False: No Machines are created; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f79187224c0f6c108a58dbd8560ccfad6738548d608f176374514e987252a6eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7124/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104613528387194880","occurred_at":"2026-09-28T16:45:41Z","failed_at":"e2e","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:615]: expected node pool nodes to report at least one release image ref before downgrade"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["64c2d1d9234c34a6135ae91e01124342e7ff5dd9dc88a4e82dbf77091dc128de","97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","b2ed02918fbe605f5650b4b12180f78e9cff28da794fad98a9e5cfcbe1ec8f13","b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","d767e2ca9d03f1dc0e13a444db797a0616296488c51bf10639db55d1b9966004"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104608138077409280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T16:24:16Z","started_at":"2026-09-28T16:24:16Z","completed_at":"2026-09-28T17:02:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T17:09:36.979931856Z","timing_metadata_checked_at":"2026-09-28T17:09:36.979931856Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T17:09:36.982649347Z","region_metadata_checked_at":"2026-09-28T17:09:36.982649347Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"91e9b98ac516c4667ac62612629d4c846b809853ae286e7f83b6158b8169b8f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104608138077409280","occurred_at":"2026-09-28T16:24:16Z","failed_at":"provision","signature_id":"f27f95b87b433141f34ec0551accb984fb054d19f57fbff11ef1f0f0955767f5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T16:48:46.474Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T16:56:22.318Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T16:48:46.48313589Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T16:48:48.126456101Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T16:48:48.218251838Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316196451Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"466F265DD247AC33\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:48:51.5604197Z\\\",\\\"duration\\\":\\\"PT7M11.5958475S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316251615Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"C2707792104BB37B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:54:09.775917Z\\\",\\\"duration\\\":\\\"PT1M38.4094883S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316265115Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"ED6706B9C7B9A085\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:54:09.775917Z\\\",\\\"duration\\\":\\\"PT1M54.5772258S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316275576Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"9BFCDEDEF69AC484\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T16:54:09.775917Z\\\",\\\"duration\\\":\\\"PT1M20.8551291S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"3\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32404980-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32404980-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316323959Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A08358469F613804\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:54:09.775917Z\\\",\\\"duration\\\":\\\"PT1M37.4251788S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.31633432Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"F3D7EC585ED8A035\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:48:51.5604197Z\\\",\\\"duration\\\":\\\"PT5M18.0784864S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:56:22.316354711Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7409280-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j7409280-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109927570685394\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j7409280-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32404980-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7409280-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32404980-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m35.833223151s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104606174522380288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7200,"pr_state":"open","pr_sha":"8ff68711751589a6d0d483b9c79e566891cc0e7c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-28T16:16:28Z","started_at":"2026-09-28T16:16:28Z","completed_at":"2026-09-28T18:35:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:42:57.957315316Z","timing_metadata_checked_at":"2026-09-28T18:42:57.957315316Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:42:57.961447703Z","region_metadata_checked_at":"2026-09-28T18:42:57.961447703Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605937833611264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7074,"pr_state":"open","pr_sha":"d68eeeb0fbe326b55631f88b73b193dece036475","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T16:15:31Z","started_at":"2026-09-28T16:15:31Z","completed_at":"2026-09-28T16:56:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T17:09:36.848103175Z","timing_metadata_checked_at":"2026-09-28T17:09:36.848103175Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T17:09:36.851329765Z","region_metadata_checked_at":"2026-09-28T17:09:36.851329765Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bd470f8ba0cf042e0d8ffd06dbd3112abbe277d9bcdc65c9d641d766d515a5be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7074/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605937833611264","occurred_at":"2026-09-28T16:15:31Z","failed_at":"provision","signature_id":"e15b79721db530238062d8eb5258b0f475f5b9ee91938ada88d2f06997fe9fd9","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T16:41:53.539Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T16:50:00.699Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T16:41:53.548987458Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T16:41:55.657571892Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T16:41:55.746008508Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:50:00.696830163Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"6AAA4A6207C70729\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3611264-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j3611264-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:48:02.4739444Z\\\",\\\"duration\\\":\\\"PT1M21.6334503S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:50:00.696877604Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"68B6F701A0C34E31\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3611264-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j3611264-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:48:02.4739444Z\\\",\\\"duration\\\":\\\"PT1M53.575039S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:50:00.696896904Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"3B3E9A86D87CA201\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3611264-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j3611264-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T16:48:02.4739444Z\\\",\\\"duration\\\":\\\"PT34.2323713S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-64061740-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-64061740-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T16:50:00.696947105Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"9C0D580618E74A6E\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3611264-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j3611264-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:48:02.4739444Z\\\",\\\"duration\\\":\\\"PT1M21.1193253S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:50:00.696958195Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2D48FCCF3470DD4C\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3611264-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j3611264-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:41:59.7167587Z\\\",\\\"duration\\\":\\\"PT7M55.4608511S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:50:00.696967625Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"BA32557B18EB08A6\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3611264-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:41:59.7167587Z\\\",\\\"duration\\\":\\\"PT6M2.706804S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:50:00.696983806Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j3611264-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j3611264-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3611264-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109931695561206\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j3611264-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-64061740-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j3611264-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-64061740-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"8m7.148003158s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605657771544576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7104,"pr_state":"closed","pr_sha":"96fefec7020ace2443c32f7f522239a74f96b4d2","final_merged_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T16:14:25Z","started_at":"2026-09-28T16:14:25Z","completed_at":"2026-09-28T18:42:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:53:18.355647072Z","timing_metadata_checked_at":"2026-09-28T18:53:18.355647072Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:53:18.359106861Z","region_metadata_checked_at":"2026-09-28T18:53:18.359106861Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e3324c34bcfb0053463238bb76aade71e2a39cf718f55312f33a7141b1b6922a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605657771544576","occurred_at":"2026-09-28T16:14:25Z","failed_at":"alert","signature_id":"fd2822c3f7f785cbd223c73c54fcf972a98b49ae3cb4a8e06c0beb83985fde95","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T17:15:52Z\n Ended: 2026-09-28T17:50:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1544576-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1544576-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605288744095744","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7221,"pr_state":"closed","pr_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","final_merged_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T16:12:53Z","started_at":"2026-09-28T16:12:53Z","completed_at":"2026-09-28T18:39:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:53:18.722209808Z","timing_metadata_checked_at":"2026-09-28T18:53:18.722209808Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:53:18.724966799Z","region_metadata_checked_at":"2026-09-28T18:53:18.724966799Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"50f8888a36c8f2b1b52f5411f668d349742fd73d7e3b45944767b8dccd999f17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605288744095744","occurred_at":"2026-09-28T16:12:53Z","failed_at":"e2e","signature_id":"75a042124b3e99ba945259807210552155a890c136bb34b4fa60646036997e91","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bltph8\" for cluster \"cluster-candidate-4-22-bltph8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02a80\u003e: \n failed to create NodePool np-bltph8: failed waiting for nodepool=\"np-bltph8\" for cluster \"cluster-candidate-4-22-bltph8\" in resourcegroup=\"rg-candidate-4-22-v77qxtrndwz7\" to finish creating: GET https://rp.j4095744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/101f36de-888a-478f-8ea5-a713440e800f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/101f36de-888a-478f-8ea5-a713440e800f\",\n \"name\": \"101f36de-888a-478f-8ea5-a713440e800f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T17:28:01.310212925Z\",\n \"endTime\": \"2026-09-28T17:47:02.14138071Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c7x6l7q7x4e1a8a-np-bltph8-pxvb4-5j9tr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-v77qxtrndwz7-candidate-bltph8--managed/providers/Microsoft.Compute/virtualMachines/c7x6l7q7x4e1a8a-np-bltph8-pxvb4-5j9tr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c7x6l7q7x4e1a8a-np-bltph8-pxvb4-5j9tr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-v77qxtrndwz7-candidate-bltph8--managed/providers/Microsoft.Compute/virtualMachines/c7x6l7q7x4e1a8a-np-bltph8-pxvb4-5j9tr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bltph8\" for cluster \"cluster-candidate-4-22-bltph8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02a80\u003e: \n failed to create NodePool np-bltph8: failed waiting for nodepool=\"np-bltph8\" for cluster \"cluster-candidate-4-22-bltph8\" in resourcegroup=\"rg-candidate-4-22-v77qxtrndwz7\" to finish creating: GET https://rp.j4095744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/101f36de-888a-478f-8ea5-a713440e800f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/101f36de-888a-478f-8ea5-a713440e800f\",\n \"name\": \"101f36de-888a-478f-8ea5-a713440e800f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T17:28:01.310212925Z\",\n \"endTime\": \"2026-09-28T17:47:02.14138071Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine c7x6l7q7x4e1a8a-np-bltph8-pxvb4-5j9tr: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-v77qxtrndwz7-candidate-bltph8--managed/providers/Microsoft.Compute/virtualMachines/c7x6l7q7x4e1a8a-np-bltph8-pxvb4-5j9tr to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c7x6l7q7x4e1a8a-np-bltph8-pxvb4-5j9tr: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-22-v77qxtrndwz7-candidate-bltph8--managed/providers/Microsoft.Compute/virtualMachines/c7x6l7q7x4e1a8a-np-bltph8-pxvb4-5j9tr to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605132682432512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7020,"pr_state":"closed","pr_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","final_merged_sha":"6cb0492059ca17ab9fe7fffef3fab0908d31e190","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T16:12:19Z","started_at":"2026-09-28T16:12:19Z","completed_at":"2026-09-28T18:29:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:32:28.523442377Z","timing_metadata_checked_at":"2026-09-28T18:32:28.523442377Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:32:28.530661053Z","region_metadata_checked_at":"2026-09-28T18:32:28.530661053Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0cb27bd608c332d830091f9c86a638ea96e0dd7740dfc0d11acd2e6d688ba859","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605132682432512","occurred_at":"2026-09-28T16:12:19Z","failed_at":"alert","signature_id":"b68e00c86c872a4db070db835721b53859e77302e16ae329d88531395a8e4cfb","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T17:11:53Z\n Ended: 2026-09-28T17:44:56Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2432512-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2432512-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"4a2aa93361ea74f972f51d3573dc88b7b558feced0bb14f4763fe480308e80ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7020/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104605132682432512","occurred_at":"2026-09-28T16:12:19Z","failed_at":"e2e","signature_id":"1cca4c55cd8817a60cc8a873fae0fecf46a8fe0f006a5fe9f16d6906ad65f148","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00080dc20\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n - pod pull-secret-test container pull-test: ContainerCreating ()\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:286]: failed to wait for image from registry.redhat.io to be pulled successfully with the added pull secret\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00080dc20\u003e: \n VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;\n - pod pull-secret-test container pull-test: ContainerCreating ()\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2ea683689d213a80fd9693db9b372fae798477a56e7e530874ac1f5a4eddf2fd","failure_pattern":"VerifyImagePulled(registry.redhat.io/ubi-minimal) timed out after 3m0.001s: expected image from registry.redhat.io/ubi-minimal in namespace pullsecret-image-test: 1 pod(s), no matching container with ImageID yet;","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2ea683689d213a80fd9693db9b372fae798477a56e7e530874ac1f5a4eddf2fd","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104593036766875648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7224,"pr_state":"closed","pr_sha":"a6cabcda34fdee6066dd2b496192fcb45072451a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T15:24:16Z","started_at":"2026-09-28T15:24:16Z","completed_at":"2026-09-28T17:51:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:02:00.851507527Z","timing_metadata_checked_at":"2026-09-28T18:02:00.851507527Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:02:00.854892016Z","region_metadata_checked_at":"2026-09-28T18:02:00.854892016Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6e563b284a682847f19d484a869de9990712e8d62f5b3e219a21a3f5f474fa45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7224/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104593036766875648","occurred_at":"2026-09-28T15:24:16Z","failed_at":"alert","signature_id":"1fcca37ac24d8001a48be0c5ac838fc8ff91d02e6a09bfd794ed9b778404688a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T16:52:47Z\n Ended: 2026-09-28T17:00:49Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6875648-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6875648-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592057438834688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7106,"pr_state":"closed","pr_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","final_merged_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T15:20:22Z","started_at":"2026-09-28T15:20:22Z","completed_at":"2026-09-28T17:51:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:02:00.373865327Z","timing_metadata_checked_at":"2026-09-28T18:02:00.373865327Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:02:00.376677418Z","region_metadata_checked_at":"2026-09-28T18:02:00.376677418Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592056927129600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T15:20:22Z","started_at":"2026-09-28T15:20:22Z","completed_at":"2026-09-28T16:18:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:27:01.158220227Z","timing_metadata_checked_at":"2026-09-28T16:27:01.158220227Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:27:01.164499407Z","region_metadata_checked_at":"2026-09-28T16:27:01.164499407Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"f2574e686324af26c66662ba10703c3f056169abc4184399bb6909f0b8a78d06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592056927129600","occurred_at":"2026-09-28T15:20:22Z","failed_at":"provision","signature_id":"245fc686bb329687b97bc133024cbedcbb44cd86f8d1e1e47b118dbf570d5a36","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T15:57:59.321Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T16:10:11.146Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T15:57:59.330300309Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T15:58:01.390991087Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T15:58:01.696498958Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:10:11.14280632Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"0311F1794B8012FE\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7129600-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j7129600-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T15:58:08.4615784Z\\\",\\\"duration\\\":\\\"PT11M40.7953789S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:10:11.142886205Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"5EA7D48F767F6548\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7129600-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j7129600-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T16:07:54.7264908Z\\\",\\\"duration\\\":\\\"PT1M38.5981419S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"1\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32015437-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32015437-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T16:10:11.14296792Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"6E7B2AA04B9A4637\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7129600-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j7129600-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:07:54.7264908Z\\\",\\\"duration\\\":\\\"PT1M22.2679526S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:10:11.142985571Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"AF3F03D52745E13A\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7129600-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j7129600-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:07:54.7264908Z\\\",\\\"duration\\\":\\\"PT1M53.5116434S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:10:11.143004953Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"9ABBF5673F832862\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7129600-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j7129600-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:07:54.7264908Z\\\",\\\"duration\\\":\\\"PT1M21.0849543S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:10:11.143022244Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"43ED8335F40DFA22\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7129600-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T15:58:08.4615784Z\\\",\\\"duration\\\":\\\"PT9M46.192924S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:10:11.143051885Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7129600-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7129600-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7129600-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109958033794530\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7129600-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32015437-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7129600-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-32015437-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"1\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"12m11.812755266s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592056415424512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7215,"pr_state":"open","pr_sha":"4c3b3d5d645acb69fa7458f18c9319f14d4142d1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T15:20:22Z","started_at":"2026-09-28T15:20:22Z","completed_at":"2026-09-28T16:14:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:27:01.281542835Z","timing_metadata_checked_at":"2026-09-28T16:27:01.281542835Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:27:01.285738722Z","region_metadata_checked_at":"2026-09-28T16:27:01.285738722Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0fd877db0e244e1728036b6cb3a203b74a691e21138705b7224b0ea6c8009404","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592056415424512","occurred_at":"2026-09-28T15:20:22Z","failed_at":"provision","signature_id":"e12ab96347b90637e243fdb23adfad98d667a0a645d1910c2ec494ebefdb4f91","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T15:59:28.426Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T16:06:34.696Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T15:59:28.435042187Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T15:59:30.367106783Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T15:59:30.626174897Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:06:34.694179167Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"4749BC6DF119391E\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5424512-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j5424512-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:04:34.5902349Z\\\",\\\"duration\\\":\\\"PT1M23.0919369S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:06:34.694242988Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"067F3DF862DEE7AA\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5424512-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j5424512-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T15:59:33.8878249Z\\\",\\\"duration\\\":\\\"PT6M54.5799485S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:06:34.694260079Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"5F03570902E1ED24\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5424512-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j5424512-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T16:04:34.5902349Z\\\",\\\"duration\\\":\\\"PT34.8973447S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-93097460-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-93097460-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T16:06:34.69431613Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A4AE1C448D11E1F8\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5424512-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j5424512-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:04:34.5902349Z\\\",\\\"duration\\\":\\\"PT1M53.053524S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:06:34.69432878Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"98C15F1801605C4C\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5424512-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j5424512-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T16:04:34.5902349Z\\\",\\\"duration\\\":\\\"PT1M21.7391537S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:06:34.69434204Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"BA6BD3DE4D4B26BF\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5424512-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T15:59:33.8878249Z\\\",\\\"duration\\\":\\\"PT5M0.6366021S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T16:06:34.694362061Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5424512-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5424512-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5424512-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109957146694709\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5424512-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-93097460-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5424512-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-93097460-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m6.259324774s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592047892598784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T15:20:20Z","started_at":"2026-09-28T15:20:20Z","completed_at":"2026-09-28T17:56:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:02:00.50660121Z","timing_metadata_checked_at":"2026-09-28T18:02:00.50660121Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:02:00.5097368Z","region_metadata_checked_at":"2026-09-28T18:02:00.5097368Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6deb7a01fdaa9af379899d08100698b3d794c5941406c5811d3f2dcb071475fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592047892598784","occurred_at":"2026-09-28T15:20:20Z","failed_at":"alert","signature_id":"01f3b1b330714082cb25db8e5c89395b0eafdbb7c5f976208821bbe7acefacea","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T16:59:48Z\n Ended: 2026-09-28T17:05:47Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2598784-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2598784-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592038186979328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T15:20:17Z","started_at":"2026-09-28T15:20:17Z","completed_at":"2026-09-28T17:49:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:02:00.163881786Z","timing_metadata_checked_at":"2026-09-28T18:02:00.163881786Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:02:00.167026176Z","region_metadata_checked_at":"2026-09-28T18:02:00.167026176Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"c6f15c77faeeceda14ed2113dd96691b3387a5ea5c2ede221098d2ff9028171d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104592038186979328","occurred_at":"2026-09-28T15:20:17Z","failed_at":"alert","signature_id":"73fe2c4208f24efcea5616c31f3812a95478add8edc71b52d609662bd139ff23","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T16:58:54Z\n Ended: 2026-09-28T17:04:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6979328-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6979328-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104591950119178240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7142,"pr_state":"closed","pr_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","final_merged_sha":"cfda26d7698257f0afa1b25f95e39ab26e8f37ef","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T15:19:56Z","started_at":"2026-09-28T15:19:56Z","completed_at":"2026-09-28T17:42:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T17:51:26.186751111Z","timing_metadata_checked_at":"2026-09-28T17:51:26.186751111Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T17:51:26.190974097Z","region_metadata_checked_at":"2026-09-28T17:51:26.190974097Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"52f779f05ba2fe4e5bb3177d7b5ab26160396204834c95186fc1c876b231ac70","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104591950119178240","occurred_at":"2026-09-28T15:19:56Z","failed_at":"e2e","signature_id":"72df9d9a52679f65dde671993fc72f0a08f84ed93d624648a2a5660cfb58bcff","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0003381e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-rvmnrkkl285p\" to finish creating: GET https://rp.j9178240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c73ad27-aa36-40fa-b776-41e8f89f246a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c73ad27-aa36-40fa-b776-41e8f89f246a\",\n \"name\": \"9c73ad27-aa36-40fa-b776-41e8f89f246a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T16:28:51.042640714Z\",\n \"endTime\": \"2026-09-28T16:47:56.610289238Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-z6d2v-wmv2z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-rvmnrkkl285p--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-z6d2v-wmv2z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-z6d2v-wmv2z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-rvmnrkkl285p--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-z6d2v-wmv2z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0003381e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-rvmnrkkl285p\" to finish creating: GET https://rp.j9178240.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c73ad27-aa36-40fa-b776-41e8f89f246a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c73ad27-aa36-40fa-b776-41e8f89f246a\",\n \"name\": \"9c73ad27-aa36-40fa-b776-41e8f89f246a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T16:28:51.042640714Z\",\n \"endTime\": \"2026-09-28T16:47:56.610289238Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20260901-np-1-z6d2v-wmv2z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-rvmnrkkl285p--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-z6d2v-wmv2z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20260901-np-1-z6d2v-wmv2z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20260901preview-rvmnrkkl285p--managed/providers/Microsoft.Compute/virtualMachines/v20260901-np-1-z6d2v-wmv2z to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e706b9ba9abb0830d83e6d6dfbf052de768f27e117863aed57d3abc9c5ff7350","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104591950119178240","occurred_at":"2026-09-28T15:19:56Z","failed_at":"alert","signature_id":"0fcfe2b08422f9a39332ea81e524fadc56ccfa65ec146684784a80ed12090466","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T16:56:14Z\n Ended: 2026-09-28T17:04:13Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9178240-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9178240-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104589634188087296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7185,"pr_state":"closed","pr_sha":"82a0c5b832cb3e6259e0c6c1090e6105a0f71753","final_merged_sha":"82a0c5b832cb3e6259e0c6c1090e6105a0f71753","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T15:10:44Z","started_at":"2026-09-28T15:10:44Z","completed_at":"2026-09-28T17:35:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T17:41:19.743501692Z","timing_metadata_checked_at":"2026-09-28T17:41:19.743501692Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T17:41:19.746991281Z","region_metadata_checked_at":"2026-09-28T17:41:19.746991281Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e7c26542f9b16204803f8228be9a1ac375e7c05f4178665aee21acdfd17c8566","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104589634188087296","occurred_at":"2026-09-28T15:10:44Z","failed_at":"e2e","signature_id":"e7ecb159e629a80b264d02c522765675f076609f52edeb1aa4e53b5b36a030c5","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-sxjtd8\" for cluster \"cluster-hs-pre-sxjtd8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001083460\u003e: \n failed to create NodePool np-sxjtd8: failed waiting for nodepool=\"np-sxjtd8\" for cluster \"cluster-hs-pre-sxjtd8\" in resourcegroup=\"rg-hs-pre--b9q689nq2wdz\" to finish creating: GET https://rp.j8087296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8a6222c-2fba-4e87-a303-c9f2805ff8b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8a6222c-2fba-4e87-a303-c9f2805ff8b9\",\n \"name\": \"e8a6222c-2fba-4e87-a303-c9f2805ff8b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T16:22:48.273113179Z\",\n \"endTime\": \"2026-09-28T16:41:51.753830234Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y0n4l3m3f1d2x9m-np-sxjtd8-978kp-mhmdc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--b9q689nq2wdz-sxjtd8--managed/providers/Microsoft.Compute/virtualMachines/y0n4l3m3f1d2x9m-np-sxjtd8-978kp-mhmdc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y0n4l3m3f1d2x9m-np-sxjtd8-978kp-mhmdc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--b9q689nq2wdz-sxjtd8--managed/providers/Microsoft.Compute/virtualMachines/y0n4l3m3f1d2x9m-np-sxjtd8-978kp-mhmdc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-sxjtd8\" for cluster \"cluster-hs-pre-sxjtd8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001083460\u003e: \n failed to create NodePool np-sxjtd8: failed waiting for nodepool=\"np-sxjtd8\" for cluster \"cluster-hs-pre-sxjtd8\" in resourcegroup=\"rg-hs-pre--b9q689nq2wdz\" to finish creating: GET https://rp.j8087296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8a6222c-2fba-4e87-a303-c9f2805ff8b9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8a6222c-2fba-4e87-a303-c9f2805ff8b9\",\n \"name\": \"e8a6222c-2fba-4e87-a303-c9f2805ff8b9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T16:22:48.273113179Z\",\n \"endTime\": \"2026-09-28T16:41:51.753830234Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y0n4l3m3f1d2x9m-np-sxjtd8-978kp-mhmdc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--b9q689nq2wdz-sxjtd8--managed/providers/Microsoft.Compute/virtualMachines/y0n4l3m3f1d2x9m-np-sxjtd8-978kp-mhmdc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y0n4l3m3f1d2x9m-np-sxjtd8-978kp-mhmdc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--b9q689nq2wdz-sxjtd8--managed/providers/Microsoft.Compute/virtualMachines/y0n4l3m3f1d2x9m-np-sxjtd8-978kp-mhmdc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104588552477085696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T15:06:26Z","started_at":"2026-09-28T15:06:26Z","completed_at":"2026-09-28T17:54:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T18:02:00.130429791Z","timing_metadata_checked_at":"2026-09-28T18:02:00.130429791Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T18:02:00.134315379Z","region_metadata_checked_at":"2026-09-28T18:02:00.134315379Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"6dcf275982e65e3564f6f2ba50b575fc89dcea2c0dc0e93f8bc5aa1770a5c5af","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104588552477085696","occurred_at":"2026-09-28T15:06:26Z","failed_at":"alert","signature_id":"8bd6e6bf74442257ea4a8d5df10532bf229f7a19b2e38c9f2d2d23e12d0aa5a0","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T16:57:44Z\n Ended: 2026-09-28T17:03:45Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7085696-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7085696-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104587545261117440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7214,"pr_state":"closed","pr_sha":"8c2b1ca82ef1ce489eda21cc7f11b6d7473c6941","final_merged_sha":"2ae778845f66d98316f4f41951553bf09a1e19db","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T15:02:25Z","started_at":"2026-09-28T15:02:25Z","completed_at":"2026-09-28T17:44:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T17:51:26.454306923Z","timing_metadata_checked_at":"2026-09-28T17:51:26.454306923Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T17:51:26.459389406Z","region_metadata_checked_at":"2026-09-28T17:51:26.459389406Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4c48a614abbdb95d28361a41dc37a234fe0c6c897e3102ac0fa8df3771fe5445","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104587545261117440","occurred_at":"2026-09-28T15:02:25Z","failed_at":"alert","signature_id":"bb22d546fa0f1214705bb254ddf09be8608855766ac347879263e53fc41c3caf","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T16:53:13Z\n Ended: 2026-09-28T16:59:14Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1117440-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1117440-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104586688620662784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7143,"pr_state":"open","pr_sha":"4cd40e7e4c9b4d024e663ff6af26b2f512e9b46f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:59:02Z","started_at":"2026-09-28T14:59:02Z","completed_at":"2026-09-28T17:38:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T17:51:26.305392217Z","timing_metadata_checked_at":"2026-09-28T17:51:26.305392217Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T17:51:26.312125495Z","region_metadata_checked_at":"2026-09-28T17:51:26.312125495Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"12ef483dd7133701f2a14f11fd23b42abbacb82e3249bf79fa2caf46247dff8b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104586688620662784","occurred_at":"2026-09-28T14:59:02Z","failed_at":"alert","signature_id":"856bdf33775551f2ab32007205cf5d6ea2f3c3f28ec94c1bfd6fbd6d4df71cdc","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T16:38:15Z\n Ended: 2026-09-28T16:46:11Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0662784-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0662784-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"4e2307d8083c684dcd3645b0db66178f7442ccc52f3221c80d53516892781460","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104586688620662784","occurred_at":"2026-09-28T14:59:02Z","failed_at":"alert","signature_id":"4ac2437ca401e1dc8a6a2e6d5b4ef3c49472cacbbbc7c08cd1239a7c62a94166","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T16:20:58Z\n Ended: 2026-09-28T16:27:58Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0662784-svc\", component=\"backend\", name=\"clusterresources\", region=\"centralus\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"78010eb3b334e1f07a75549fbb15487a8ddd3a8a9337b6bff39ff65465df7bd2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7143/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104586688620662784","occurred_at":"2026-09-28T14:59:02Z","failed_at":"alert","signature_id":"a0f3783cbc9752f6f287380baf21644ed565dc45c0b9da951cea46157a26275a","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 39 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-qwlf6\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-qwlf6 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-w28wh\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-w28wh has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-zxx8r\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-zxx8r has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-g76qj\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-g76qj has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-d9qzz\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-d9qzz has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-fpkv2\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-fpkv2 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-cwzx8\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-cwzx8 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-qm6c8\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-qm6c8 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-4996z\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-4996z has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-675f5\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-675f5 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-gd9k9\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-gd9k9 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-z58gp\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-z58gp has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-4j2hv\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-4j2hv has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-8n4fj\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-8n4fj has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-8q8gs\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-8q8gs has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-ddxpz\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-ddxpz has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-9kqwf\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-9kqwf has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-tpk28\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-tpk28 has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-wpwxf\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-wpwxf has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-c9pkv\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-c9pkv has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-r2z8h\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-r2z8h has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-v6lst\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-v6lst has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-6nc22\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-6nc22 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-52zbx\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-52zbx has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-j8nbh\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-j8nbh has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-f9lhx\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-f9lhx has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-dp559\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-dp559 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-j4pwc\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-j4pwc has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-p9w7c\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-p9w7c has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-hqdp9\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-hqdp9 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-dgb65\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-dgb65 has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-rvk8h\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-rvk8h has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-mpfhk\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-mpfhk has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-qm2ts\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-qm2ts has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-27cpw\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-27cpw has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-mxsvz\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-mxsvz has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-hgtjs\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-hgtjs has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-c8pzf\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-c8pzf has been in a non-ready state for longer than 5 minutes.\nFiring 39:\n State: Fired (not resolved)\n Started: 2026-09-28T16:03:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0662784-svc\", component=\"kubernetes-infrastructure\", namespace=\"monitoring\", pod=\"kube-events-7bcbfb996f-dvzkd\", region=\"centralus\", severity=\"warning\"\n Description: Pod monitoring/kube-events-7bcbfb996f-dvzkd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104582134038532096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6869,"pr_state":"open","pr_sha":"4058f9eba8fc7b0fb6338eb0433682c6448de89d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:40:48Z","started_at":"2026-09-28T14:40:48Z","completed_at":"2026-09-28T15:12:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:24:01.982534468Z","timing_metadata_checked_at":"2026-09-28T15:24:01.982534468Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:24:01.986048957Z","region_metadata_checked_at":"2026-09-28T15:24:01.986048957Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e14d652483a000ba40027f71c6df592e725d2a0e4b470829fd10bedba749a053","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104582134038532096","occurred_at":"2026-09-28T14:40:48Z","failed_at":"provision","signature_id":"fa17931981292f423927883a89b52aa95c48fed87ff6ab8291b372cd88031f32","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T15:01:37.564Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n Kind: ARM\\n Template: templates/svc-cluster.bicep\\n Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-09-28T15:01:39.047Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-09-28T15:01:41.809Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=07143feb5a4416edfd80f78e9bd52b0ace9e0b4259fb3b361ffbc72b09db2a4e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j8532096-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F07143feb5a4416edfd80f78e9bd52b0ace9e0b4259fb3b361ffbc72b09db2a4e\ntime=2026-09-28T15:06:13.242Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8532096-svc/providers/Microsoft.Resources/deployments/07143feb5a4416edfd80f78e9bd52b0ace9e0b4259fb3b361ffbc72b09db2a4e/operationStatuses/08584109991856512016\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8532096-svc/providers/Microsoft.Resources/deployments/cluster-fgxbml5ybyqlw\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8532096-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j8532096-svc\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8532096-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-system-29994265-vmss'. Please try alternative sizes 'Standard_D4as_v7' (zones 3, 1, 2), 'Standard_D4ads_v7' (zones 1, 2, 3), 'Standard_E4as_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j8532096-svc-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-system-29994265-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"2\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104582134726397952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6869,"pr_state":"open","pr_sha":"4058f9eba8fc7b0fb6338eb0433682c6448de89d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:40:47Z","started_at":"2026-09-28T14:40:47Z","completed_at":"2026-09-28T14:40:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:14:01.480402274Z","timing_metadata_checked_at":"2026-09-28T15:14:01.480402274Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-28T15:14:01.483652864Z","region_metadata_checked_at":"2026-09-28T16:16:56.333800717Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"34557c0387065e6274212100a7ae61bf7bce49b6d97c5e343242ba150ef96202","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104582134726397952","occurred_at":"2026-09-28T14:40:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104581730311606272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6577,"pr_state":"closed","pr_sha":"0e7b6ab211d89384262c214620c9a6861ad367b4","final_merged_sha":"3d03a40d19bf9b85b85165f4f9432cbbcf37480e","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-28T14:39:08Z","started_at":"2026-09-28T14:39:08Z","completed_at":"2026-09-28T16:50:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:58:44.059496464Z","timing_metadata_checked_at":"2026-09-28T16:58:44.059496464Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:58:44.063181653Z","region_metadata_checked_at":"2026-09-28T16:58:44.063181653Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104579857332572160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7200,"pr_state":"open","pr_sha":"c0bdffdc6975529f06e36d7020e4a708661335c4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:31:53Z","started_at":"2026-09-28T14:31:53Z","completed_at":"2026-09-28T15:11:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:24:02.431112951Z","timing_metadata_checked_at":"2026-09-28T15:24:02.431112951Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:24:02.434319941Z","region_metadata_checked_at":"2026-09-28T15:24:02.434319941Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"dd73a2a8d65d79501fe5b1f065e10308b55f8fccb08f93b6fa5bb7105a5d8dd6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104579857332572160","occurred_at":"2026-09-28T14:31:53Z","failed_at":"provision","signature_id":"5b0cccc265d8272c63c2015099aee3246e496694b91725d91dd789992650d9f5","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T14:56:44.121Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T15:04:20.182Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T14:56:44.133408984Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T14:56:45.999747591Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T14:56:46.087122797Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T15:04:20.179772906Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"6904C144EA6E5071\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j2572160-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j2572160-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T15:02:20.3817346Z\\\",\\\"duration\\\":\\\"PT1M58.3746655S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T15:04:20.179836397Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A4BA1CEE66306436\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j2572160-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j2572160-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T15:02:20.3817346Z\\\",\\\"duration\\\":\\\"PT1M37.5722982S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T15:04:20.179850277Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"410417A2FC6329C6\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j2572160-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j2572160-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T14:56:49.1631026Z\\\",\\\"duration\\\":\\\"PT7M8.6010377S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-17691843-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-17691843-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T15:04:20.179898728Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"70FD9837C10AD01B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j2572160-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j2572160-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T15:02:20.3817346Z\\\",\\\"duration\\\":\\\"PT1M21.3213377S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T15:04:20.179909679Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"8940E8E2A38F1650\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j2572160-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j2572160-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T15:02:20.3817346Z\\\",\\\"duration\\\":\\\"PT1M52.0121162S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T15:04:20.179919029Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"01D23ED14CA92F00\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j2572160-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T14:56:49.1631026Z\\\",\\\"duration\\\":\\\"PT5M31.1376288S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T15:04:20.179947539Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j2572160-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j2572160-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2572160-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584109994792416073\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j2572160-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-17691843-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2572160-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-17691843-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m36.046545346s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104578384683077632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7181,"pr_state":"closed","pr_sha":"9be5da800bb3fb9bc4e61776d44ed6af5efb0ea8","final_merged_sha":"3ae51eab9b946eb1b94407de9646f52a67175cd0","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:26:01Z","started_at":"2026-09-28T14:26:01Z","completed_at":"2026-09-28T16:42:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:47:50.954734919Z","timing_metadata_checked_at":"2026-09-28T16:47:50.954734919Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:47:50.957314311Z","region_metadata_checked_at":"2026-09-28T16:47:50.957314311Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"63362422f6c1547ca4534d126827a1d5fdc50cc541699511f67c825c2b3f87b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7181/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104578384683077632","occurred_at":"2026-09-28T14:26:01Z","failed_at":"e2e","signature_id":"ac47319bea43a8ae82a11fa35dcd2a3b65d8f02c99e7937378a00807512c62aa","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:243]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullhctmh8kwn4gw.azurecr.io\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011dc00\u003e: \n VerifyImagePulled(acrpullhctmh8kwn4gw.azurecr.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:243]: [day 2] pod in namespace acr-pull-test-day2 never pulled image ubi-minimal from private ACR acrpullhctmh8kwn4gw.azurecr.io\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00011dc00\u003e: \n VerifyImagePulled(acrpullhctmh8kwn4gw.azurecr.io/ubi-minimal) timed out after 3m0.008s: failed to list pods in namespace acr-pull-test-day2: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576830420815872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7202,"pr_state":"closed","pr_sha":"bf798191f1e4784ba8c222bbae084b193940b0ff","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:19:52Z","started_at":"2026-09-28T14:19:52Z","completed_at":"2026-09-28T16:36:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:47:51.727371462Z","timing_metadata_checked_at":"2026-09-28T16:47:51.727371462Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:47:51.730165953Z","region_metadata_checked_at":"2026-09-28T16:47:51.730165953Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"0c490d18e0e53df57fb21740b37fec48d04ad30e528a3629b583224d4d1ea265","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576830420815872","occurred_at":"2026-09-28T14:19:52Z","failed_at":"alert","signature_id":"b45324aaec7eb7902136101858f41a6bec16eed34781ffde48402ce2ed72f330","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T15:42:33Z\n Ended: 2026-09-28T15:48:31Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0815872-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0815872-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576818622238720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7214,"pr_state":"closed","pr_sha":"348364a54559dcede0e74a9e4e09def229be2488","final_merged_sha":"2ae778845f66d98316f4f41951553bf09a1e19db","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:19:49Z","started_at":"2026-09-28T14:19:49Z","completed_at":"2026-09-28T14:27:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:14:02.039713666Z","timing_metadata_checked_at":"2026-09-28T15:14:02.039713666Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-28T15:14:02.042365057Z","region_metadata_checked_at":"2026-09-28T16:16:56.521290298Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"7cf696c0a27f994bb964c99d2976da0bf4519e164b59789236349e5d3f71d35a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576818622238720","occurred_at":"2026-09-28T14:19:49Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576814264356864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T14:19:46Z","started_at":"2026-09-28T14:19:46Z","completed_at":"2026-09-28T14:41:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T14:53:20.830351264Z","timing_metadata_checked_at":"2026-09-28T14:53:20.830351264Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-28T14:53:20.833519054Z","region_metadata_checked_at":"2026-09-28T15:55:24.805483416Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6d302bd47a2d22432472c98590a8698e2b2f3ddf411f86b14f64ce0e7f894da5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576814264356864","occurred_at":"2026-09-28T14:19:46Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576809239580672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:19:41Z","started_at":"2026-09-28T14:19:41Z","completed_at":"2026-09-28T14:41:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T14:53:20.463530711Z","timing_metadata_checked_at":"2026-09-28T14:53:20.463530711Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T14:53:20.468875694Z","region_metadata_checked_at":"2026-09-28T14:53:20.468875694Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fdea19e9879f194b50c2765d140f31997914595e4990174246c2fc6c4bc3b351","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576809239580672","occurred_at":"2026-09-28T14:19:41Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576806710415360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7215,"pr_state":"open","pr_sha":"4c3b3d5d645acb69fa7458f18c9319f14d4142d1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:19:39Z","started_at":"2026-09-28T14:19:39Z","completed_at":"2026-09-28T15:03:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:14:02.169798945Z","timing_metadata_checked_at":"2026-09-28T15:14:02.169798945Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:14:02.172267437Z","region_metadata_checked_at":"2026-09-28T15:14:02.172267437Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"de65adcb33b46c5212809990f250f006f82d28c7ec359a676bab0f72b5daf4ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576806710415360","occurred_at":"2026-09-28T14:19:39Z","failed_at":"provision","signature_id":"596d124e078a20a425d61e9930296af8fedf5e0099dcc740de0f522cfa8668af","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T14:47:30.090Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T14:54:36.446Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T14:47:30.098885868Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T14:47:31.998596058Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T14:47:32.096190187Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T14:54:36.443742575Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"7422A9B9C0E00015\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0415360-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j0415360-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T14:52:06.8114069Z\\\",\\\"duration\\\":\\\"PT1M22.9729442S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T14:54:36.443804706Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"A288211471CD0A68\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0415360-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j0415360-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T14:52:06.8114069Z\\\",\\\"duration\\\":\\\"PT2M25.5163011S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"1\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-40607818-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-40607818-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T14:54:36.443859458Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"67AAD81D54389E82\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0415360-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j0415360-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T14:47:35.7788484Z\\\",\\\"duration\\\":\\\"PT6M26.5731388S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T14:54:36.443870708Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"E019F45794232502\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0415360-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j0415360-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T14:52:06.8114069Z\\\",\\\"duration\\\":\\\"PT1M52.7194683S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T14:54:36.443881869Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"BF186E56B03A4C4B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0415360-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j0415360-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T14:52:06.8114069Z\\\",\\\"duration\\\":\\\"PT1M36.8059515S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T14:54:36.443891779Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"532DC9DFA113CABD\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0415360-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T14:47:35.7788484Z\\\",\\\"duration\\\":\\\"PT4M30.9311267S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T14:54:36.44391733Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j0415360-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j0415360-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0415360-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584110000332060255\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j0415360-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-40607818-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j0415360-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-40607818-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"1\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m6.345035722s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576804189638656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7203,"pr_state":"open","pr_sha":"75d72389678ef3781f99923a095f0f5dfe0094f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:19:37Z","started_at":"2026-09-28T14:19:37Z","completed_at":"2026-09-28T15:05:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:14:01.906094198Z","timing_metadata_checked_at":"2026-09-28T15:14:01.906094198Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:14:01.908669289Z","region_metadata_checked_at":"2026-09-28T15:14:01.908669289Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"bf24871bee42c2cd01695169ae4d5415754ff455689652c51a1734ddc1def935","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576804189638656","occurred_at":"2026-09-28T14:19:37Z","failed_at":"provision","signature_id":"aea56e1c6bff3ce711725997c0e6216095b05913193456533d76f637a946367a","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T14:59:06.467Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-28T14:59:07.052Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:bd6cc099-ebdb-45c9-b00c-0802e69adff6 cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-ci01-j9638656] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j9638656.hcpsvc.osadev.cloud] image:map[digest:sha256:e46e58946095455f1186c85136cc576df37e30746f4cbf6630f40ee15e5c51e6 registry:arohcpsvcdev.azurecr.io repository:ci-op-w348pxjm/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-w348pxjm/pipeline:pull workloadIdentityClientId:78d6364f-6f7b-4fee-9060-b833e4581820 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:85e8e804-1ab2-4c98-b7ae-b44472a84e8a workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-09-28T14:59:07.052Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-28T14:59:07.261Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-09-28T14:59:07.573Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-09-28T14:59:07.573Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-28T14:59:07.688Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-28T14:59:15.529Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-09-28T14:59:15.529Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40d94d1862472748004cc33c4c935e1c5142aa767d3a1a077dd2606d1dc7c7d3","failure_pattern":"failed to roll out Helm release: Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["40d94d1862472748004cc33c4c935e1c5142aa767d3a1a077dd2606d1dc7c7d3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576709901684736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"e7500a3658c1692b6c32705cb7edf403b704aae4","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T14:19:23Z","started_at":"2026-09-28T14:19:23Z","completed_at":"2026-09-28T16:30:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:37:05.552991126Z","timing_metadata_checked_at":"2026-09-28T16:37:05.552991126Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:37:05.557683413Z","region_metadata_checked_at":"2026-09-28T16:37:05.557683413Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"45242414cc9389c9483b02692b2269508344281f486da85183224d6d4d0f6dc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576709901684736","occurred_at":"2026-09-28T14:19:23Z","failed_at":"alert","signature_id":"637d9f518ac8eebc8d39c5a4b7ac660a4f39e458e3c7f2cb4b55812a9f97d2ed","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T15:37:38Z\n Ended: 2026-09-28T15:43:41Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1684736-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1684736-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104576662317305856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6226,"pr_state":"closed","pr_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","final_merged_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T14:19:12Z","started_at":"2026-09-28T14:19:12Z","completed_at":"2026-09-28T16:39:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:47:50.658404361Z","timing_metadata_checked_at":"2026-09-28T16:47:50.658404361Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:47:50.662288249Z","region_metadata_checked_at":"2026-09-28T16:47:50.662288249Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104569503722508288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"closed","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T13:50:19Z","started_at":"2026-09-28T13:50:19Z","completed_at":"2026-09-28T14:00:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T14:11:00.071634069Z","timing_metadata_checked_at":"2026-09-28T14:11:00.071634069Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-28T14:11:00.076780552Z","region_metadata_checked_at":"2026-09-28T15:14:01.362869054Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5c29bec9be299c66d7eace98e9a159bfd8e0eef28d6ce5bb26ebd52bbfbf2bc7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104569503722508288","occurred_at":"2026-09-28T13:50:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104564648333283328","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7225,"pr_state":"closed","pr_sha":"72e9f5a561068e3526344d3fb03a5ceee6030608","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T13:31:27Z","started_at":"2026-09-28T13:31:27Z","completed_at":"2026-09-28T15:46:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:55:25.067976493Z","timing_metadata_checked_at":"2026-09-28T15:55:25.067976493Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:55:25.070397885Z","region_metadata_checked_at":"2026-09-28T15:55:25.070397885Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"14869b6840d36ff5b11530c72c5ad32e127b95721f5de51ba23bdbfdceb0dbe6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7225/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104564648333283328","occurred_at":"2026-09-28T13:31:27Z","failed_at":"alert","signature_id":"6588d10f1b9c7b543415a80c3d0f51e6af81c61ab7d144c6b3dce965ff9765d1","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T14:24:12Z\n Ended: 2026-09-28T14:56:14Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3283328-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3283328-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104562485683032064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7104,"pr_state":"closed","pr_sha":"5404b756a69f74f2e432703d4c0bb7e5a72701ef","final_merged_sha":"a2e35daa20b37841bf1d2f73e88d95554cdc5664","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T13:22:52Z","started_at":"2026-09-28T13:22:52Z","completed_at":"2026-09-28T15:41:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:55:24.500332674Z","timing_metadata_checked_at":"2026-09-28T15:55:24.500332674Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-09-28T15:55:24.502920666Z","region_metadata_checked_at":"2026-09-28T16:58:44.161651555Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"27074dee9f9feb76a8ffacb62c075e5c107782c3d727b8491ec91567dca2a553","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7104/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104562485683032064","occurred_at":"2026-09-28T13:22:52Z","failed_at":"e2e","signature_id":"8e7656a6acdc1e558b5e327917ebbb4231a9f78559a33ae638709b57cfbe1f81","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca2e60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-z4h7jf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-z4h7jf/operationStatuses/08584110018702456355\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name/providers/Microsoft.Authorization/roleAssignments/ce4c794c-50e9-565e-99cc-b7dd7e08c5a9' at line '1' and column '29054'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 29054,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/providers/Microsoft.Authorization/roleAssignments/e5697265-1ef6-54a3-8e48-b96178ae0611' at line '1' and column '30683'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 30683,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:82]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca2e60\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-np-labels-taints-z4h7jf\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-np-labels-taints-z4h7jf/operationStatuses/08584110018702456355\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-r6cuakilyx67k\\\",\\r\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-name/providers/Microsoft.Authorization/roleAssignments/ce4c794c-50e9-565e-99cc-b7dd7e08c5a9' at line '1' and column '29054'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 29054,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n },\\r\\n {\\r\\n \\\"code\\\": \\\"InvalidTemplate\\\",\\r\\n \\\"message\\\": \\\"Unable to process template language expressions for resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-2wjjfp5h47hw/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/providers/Microsoft.Authorization/roleAssignments/e5697265-1ef6-54a3-8e48-b96178ae0611' at line '1' and column '30683'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'\\\",\\r\\n \\\"additionalInfo\\\": [\\r\\n {\\r\\n \\\"type\\\": \\\"TemplateViolation\\\",\\r\\n \\\"info\\\": {\\r\\n \\\"lineNumber\\\": 1,\\r\\n \\\"linePosition\\\": 30683,\\r\\n \\\"path\\\": \\\"\\\"\\r\\n }\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4adf443619cbd9518f5388bac91db3b79c0430d7fca6e294ba38f40513bd0e3a","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InvalidTemplate; detail message Unable to process template language expressions for resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-name/providers/Microsoft.Authorization/roleAssignments/\u003cuuid\u003e' at line '\u003cline\u003e' and column '\u003ccolumn\u003e'. 'Language expression property 'principalId' can't be evaluated, the referencing object is null.'; provider Microsoft.Authorization","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4adf443619cbd9518f5388bac91db3b79c0430d7fca6e294ba38f40513bd0e3a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7155/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104561547119431680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7155,"pr_state":"closed","pr_sha":"b99e022318555865ffcb4306017cec183e68d902","final_merged_sha":"b99e022318555865ffcb4306017cec183e68d902","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T13:19:08Z","started_at":"2026-09-28T13:19:08Z","completed_at":"2026-09-28T15:42:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:55:24.682740302Z","timing_metadata_checked_at":"2026-09-28T15:55:24.682740302Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:55:24.685281894Z","region_metadata_checked_at":"2026-09-28T15:55:24.685281894Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104561257146224640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7054,"pr_state":"closed","pr_sha":"f0e1fa3e33ffc30bc4eeec511d85f73874d58b9b","final_merged_sha":"7553575593411fb5f3c1d57088aa752a234c76d1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T13:17:41Z","started_at":"2026-09-28T13:17:41Z","completed_at":"2026-09-28T15:44:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:55:24.343101367Z","timing_metadata_checked_at":"2026-09-28T15:55:24.343101367Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:55:24.346247058Z","region_metadata_checked_at":"2026-09-28T15:55:24.346247058Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"562d74704b0958b4af1139bdc3f2e235e095f7c056e702af0244a47e55fac3b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104561257146224640","occurred_at":"2026-09-28T13:17:41Z","failed_at":"alert","signature_id":"2849f38d4e46e23a374c8bcb9103369c6a91ad3b81860cb4691446e160ba14ef","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T14:42:44Z\n Ended: 2026-09-28T14:48:45Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6224640-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6224640-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551522074365952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7221,"pr_state":"closed","pr_sha":"bfa1d13b5bf3f2b8163b8d7653a64ff1501c6a9a","final_merged_sha":"d873d5fd809e699ac864420434c1824fcabd44c1","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T12:39:18Z","started_at":"2026-09-28T12:39:18Z","completed_at":"2026-09-28T14:51:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:03:31.817797855Z","timing_metadata_checked_at":"2026-09-28T15:03:31.817797855Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:03:31.821208544Z","region_metadata_checked_at":"2026-09-28T15:03:31.821208544Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"69bb8d65d80c75242064be4a7bb44cbdeea29fec8d4157776d7f11b43098cdc4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551522074365952","occurred_at":"2026-09-28T12:39:18Z","failed_at":"alert","signature_id":"c8b7bde7071fc9b70fbf2e0c3fd7b6a18f6e7bd846c88f638e2208d436b6bc37","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:33:38Z\n Ended: 2026-09-28T14:05:42Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4365952-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4365952-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"7d3acdd6da2ebc6171aa6235bc749260fe183d5dc40aa1c5b9c6e23033e41304","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7221/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551522074365952","occurred_at":"2026-09-28T12:39:18Z","failed_at":"e2e","signature_id":"c24e3d212804f8897ef2c81ade146070e7a5fd62032965fa1424ca72aa49afbd","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-bq4xzd\" for cluster \"cluster-hs-pre-bq4xzd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a77280\u003e: \n failed to create NodePool np-bq4xzd: failed waiting for nodepool=\"np-bq4xzd\" for cluster \"cluster-hs-pre-bq4xzd\" in resourcegroup=\"rg-hs-pre--t5k7dv9kqkn6\" to finish creating: GET https://rp.j4365952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7ac9caa-24fd-4f24-be91-2c74f17d3115\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7ac9caa-24fd-4f24-be91-2c74f17d3115\",\n \"name\": \"c7ac9caa-24fd-4f24-be91-2c74f17d3115\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:36:17.28936251Z\",\n \"endTime\": \"2026-09-28T13:55:25.887167153Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e6l4u8u5m8k2h1y-np-bq4xzd-k5mvt-twpcj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--t5k7dv9kqkn6-bq4xzd--managed/providers/Microsoft.Compute/virtualMachines/e6l4u8u5m8k2h1y-np-bq4xzd-k5mvt-twpcj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e6l4u8u5m8k2h1y-np-bq4xzd-k5mvt-twpcj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--t5k7dv9kqkn6-bq4xzd--managed/providers/Microsoft.Compute/virtualMachines/e6l4u8u5m8k2h1y-np-bq4xzd-k5mvt-twpcj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-bq4xzd\" for cluster \"cluster-hs-pre-bq4xzd\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a77280\u003e: \n failed to create NodePool np-bq4xzd: failed waiting for nodepool=\"np-bq4xzd\" for cluster \"cluster-hs-pre-bq4xzd\" in resourcegroup=\"rg-hs-pre--t5k7dv9kqkn6\" to finish creating: GET https://rp.j4365952.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7ac9caa-24fd-4f24-be91-2c74f17d3115\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7ac9caa-24fd-4f24-be91-2c74f17d3115\",\n \"name\": \"c7ac9caa-24fd-4f24-be91-2c74f17d3115\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:36:17.28936251Z\",\n \"endTime\": \"2026-09-28T13:55:25.887167153Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e6l4u8u5m8k2h1y-np-bq4xzd-k5mvt-twpcj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--t5k7dv9kqkn6-bq4xzd--managed/providers/Microsoft.Compute/virtualMachines/e6l4u8u5m8k2h1y-np-bq4xzd-k5mvt-twpcj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e6l4u8u5m8k2h1y-np-bq4xzd-k5mvt-twpcj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--t5k7dv9kqkn6-bq4xzd--managed/providers/Microsoft.Compute/virtualMachines/e6l4u8u5m8k2h1y-np-bq4xzd-k5mvt-twpcj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551505146155008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"closed","pr_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","final_merged_sha":"e772682c6ada7bf3fd5674946d72fe25f79d834b","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T12:39:14Z","started_at":"2026-09-28T12:39:14Z","completed_at":"2026-09-28T14:59:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:14:01.66425788Z","timing_metadata_checked_at":"2026-09-28T15:14:01.66425788Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:14:01.667910068Z","region_metadata_checked_at":"2026-09-28T15:14:01.667910068Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0bbc48ca78eb5cbd08aa6c0b762722b8261e27fa2cd089a53d902eed2b0a9b4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551505146155008","occurred_at":"2026-09-28T12:39:14Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"724e3b16440767ca647245da7b6f863624506ce56edcd8c098ed565294c44086","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551505146155008","occurred_at":"2026-09-28T12:39:14Z","failed_at":"e2e","signature_id":"d72030c4a0f636f0bfa24f277e9369a539321fccdfff48a303919b0b450bc268","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001373000\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k8wr4fpfcrsg\" to finish creating: GET https://rp.j6155008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf7814ee-225c-4e86-a66b-262e98fa9add\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf7814ee-225c-4e86-a66b-262e98fa9add\",\n \"name\": \"cf7814ee-225c-4e86-a66b-262e98fa9add\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:36:07.829911324Z\",\n \"endTime\": \"2026-09-28T13:55:12.891520361Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9v0q3j6a3y7e1k-np-1-cg9ct-bj9h7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-k8wr4fpfcrsg--managed/providers/Microsoft.Compute/virtualMachines/u9v0q3j6a3y7e1k-np-1-cg9ct-bj9h7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9v0q3j6a3y7e1k-np-1-cg9ct-bj9h7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-k8wr4fpfcrsg--managed/providers/Microsoft.Compute/virtualMachines/u9v0q3j6a3y7e1k-np-1-cg9ct-bj9h7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001373000\u003e: \n failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k8wr4fpfcrsg\" to finish creating: GET https://rp.j6155008.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf7814ee-225c-4e86-a66b-262e98fa9add\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf7814ee-225c-4e86-a66b-262e98fa9add\",\n \"name\": \"cf7814ee-225c-4e86-a66b-262e98fa9add\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:36:07.829911324Z\",\n \"endTime\": \"2026-09-28T13:55:12.891520361Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9v0q3j6a3y7e1k-np-1-cg9ct-bj9h7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-k8wr4fpfcrsg--managed/providers/Microsoft.Compute/virtualMachines/u9v0q3j6a3y7e1k-np-1-cg9ct-bj9h7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9v0q3j6a3y7e1k-np-1-cg9ct-bj9h7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-k8wr4fpfcrsg--managed/providers/Microsoft.Compute/virtualMachines/u9v0q3j6a3y7e1k-np-1-cg9ct-bj9h7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f5a0f86550ac4c2e2eca003b4965150a16a87892ad823a44090dba911cf10cdd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551505146155008","occurred_at":"2026-09-28T12:39:14Z","failed_at":"alert","signature_id":"0f15772b2206346fb5d44faaaa43fc1a23bbe1d33da8dea71728baa03c962358","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:58:43Z\n Ended: 2026-09-28T14:07:44Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6155008-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6155008-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551230880616448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7106,"pr_state":"closed","pr_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","final_merged_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T12:38:08Z","started_at":"2026-09-28T12:38:08Z","completed_at":"2026-09-28T14:45:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T14:53:20.656208208Z","timing_metadata_checked_at":"2026-09-28T14:53:20.656208208Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T14:53:20.658921Z","region_metadata_checked_at":"2026-09-28T14:53:20.658921Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"dd0b934a5ab586a11f8ec51d0fd4c0692f98b68da625fd05786bf56f6c231300","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104551230880616448","occurred_at":"2026-09-28T12:38:08Z","failed_at":"e2e","signature_id":"d0a0ef141598400a6c63d0737ab5c55ee05359e8d50ed563cb531e14fff81ef2","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000693320\u003e: \n DNS for route host agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:201]: failed to verify simple web app on back-level cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000693320\u003e: \n DNS for route host agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-swv95.apps.aro.u1w0b2p8i5n8f3h.2e4g.j0616448.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104548669041676288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T12:27:56Z","started_at":"2026-09-28T12:27:56Z","completed_at":"2026-09-28T14:49:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T15:03:31.598832841Z","timing_metadata_checked_at":"2026-09-28T15:03:31.598832841Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T15:03:31.601909832Z","region_metadata_checked_at":"2026-09-28T15:03:31.601909832Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"186653b98ee0bc04d94c77708f1cce65299c76b0c8b015426cfcaf44cea7f75d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104548669041676288","occurred_at":"2026-09-28T12:27:56Z","failed_at":"alert","signature_id":"d2d57b385cfb20f234fa1e1737740b5d90198f87bf051ebb2a00dd1115909d3b","test_name":"[aro-hcp-observability] [svc] alert KustoLogsDataStale does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-28T14:36:28Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j1676288-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1676288/providers/microsoft.monitor/accounts/services-j1676288\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6c86cd5988-85nd2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j1676288-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-28T14:36:28Z\n Severity: Sev3\n Labels: alertname=\"KustoLogsDataStale\", cluster=\"ci01-j1676288-svc\", component=\"kusto\", container=\"aro-hcp-exporter\", endpoint=\"metrics\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"aro-hcp-exporter\", kusto_cluster=\"hcp-dev-us-2\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1676288/providers/microsoft.monitor/accounts/services-j1676288\", namespace=\"aro-hcp-exporter\", pod=\"aro-hcp-exporter-6c86cd5988-85nd2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"aro-hcp-exporter\", severity=\"warning\", table=\"systemdlogs\"\n Description: Kusto log data for table systemdlogs on cluster ci01-j1676288-svc (Kusto cluster hcp-dev-us-2) is stale. Check the ingestion pipeline.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3","failure_pattern":"alert [svc] KustoLogsDataStale fired","search_query":"alert [svc] KustoLogsDataStale fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["7d2512936ecffffedcea4866a7c28e137e6c55cf74dfbca28922a8810e457de3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104540479675699200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T11:55:25Z","started_at":"2026-09-28T11:55:25Z","completed_at":"2026-09-28T12:26:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T12:36:52.499408452Z","timing_metadata_checked_at":"2026-09-28T12:36:52.499408452Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T12:36:52.506599328Z","region_metadata_checked_at":"2026-09-28T12:36:52.506599328Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1ceb81af6f5231338f99eea7ae91d60568b511e54ffb47b725ace2fc6c8d5db1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104540479675699200","occurred_at":"2026-09-28T11:55:25Z","failed_at":"provision","signature_id":"b2ae5eb0392e6e1bc7dcec48c107a273940a93afec94e0712cb5617629f457a9","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T12:07:41.041Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T12:15:18.546Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T12:07:41.053161868Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T12:07:43.329595506Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T12:07:43.531590773Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T12:15:18.541524871Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"AD2AC9DE0BA65092\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5699200-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j5699200-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T12:13:09.5885526Z\\\",\\\"duration\\\":\\\"PT1M22.2941476S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T12:15:18.541698705Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"795D33A61BFD5FC4\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5699200-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j5699200-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T12:13:09.5885526Z\\\",\\\"duration\\\":\\\"PT1M23.0194131S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T12:15:18.541724696Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"B79CD1DE1D75E03E\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5699200-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j5699200-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T12:13:09.5885526Z\\\",\\\"duration\\\":\\\"PT1M55.2502144S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T12:15:18.541752616Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1C6C75893BADF622\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5699200-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j5699200-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T12:13:09.5885526Z\\\",\\\"duration\\\":\\\"PT1M53.2791501S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T12:15:18.541784267Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"097072C37326C4E8\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5699200-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j5699200-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T12:07:47.6806894Z\\\",\\\"duration\\\":\\\"PT6M43.9724254S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"2\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-16955988-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-16955988-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T12:15:18.541891629Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"6469ADD3757B8F13\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5699200-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T12:07:47.6806894Z\\\",\\\"duration\\\":\\\"PT5M21.6490011S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T12:15:18.541955731Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j5699200-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j5699200-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5699200-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584110096217749999\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j5699200-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-16955988-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 3, 1, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5699200-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-16955988-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"2\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m37.488796782s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"477826b5ecb5142cd475ed1c70b738288af9806a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T11:49:34Z","started_at":"2026-09-28T11:49:34Z","completed_at":"2026-09-28T14:23:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T14:31:54.175233909Z","timing_metadata_checked_at":"2026-09-28T14:31:54.175233909Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T14:31:54.178586698Z","region_metadata_checked_at":"2026-09-28T14:31:54.178586698Z"},"failed_at":["alert","e2e"],"failed_test_count":23,"occurrences":[{"row_id":"19bfac339938cfc19740e76026d85a3ffedb8ece3ed3b640a63fb99f2be75441","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"alert","signature_id":"b254aaa1635ed84a93daf87b215afa6a9984a990ff39d417d58e92d00c86be46","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:09:10Z\n Ended: 2026-09-28T13:39:13Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j8288256-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j8288256-mgmt-2\nNamespace: ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"360f478624864f9ad18a91bbc151955c54c278f9b5b4a0dcca3160d33cbf81b5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"4f4e5752545f8f46e297ca73e9f4e1a12d4ee1ba77d68cd29f8525db53cf046a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-c89c6r\" with version 5.0.0-rc.3 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b10480\u003e: \n failed to create HCP cluster cluster-zstream-5-0-c89c6r: failed waiting for cluster=\"cluster-zstream-5-0-c89c6r\" in resourcegroup=\"rg-zstream-upgrade-5-0-6kc9p4vcvjqq\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37f813f1-0df9-4583-beb9-56f83fee5c0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37f813f1-0df9-4583-beb9-56f83fee5c0a\",\n \"name\": \"37f813f1-0df9-4583-beb9-56f83fee5c0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:45.206355596Z\",\n \"endTime\": \"2026-09-28T13:22:51.172490782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.3 is Partial (want Completed), started 26m23s ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-5-0-c89c6r\" with version 5.0.0-rc.3 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b10480\u003e: \n failed to create HCP cluster cluster-zstream-5-0-c89c6r: failed waiting for cluster=\"cluster-zstream-5-0-c89c6r\" in resourcegroup=\"rg-zstream-upgrade-5-0-6kc9p4vcvjqq\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37f813f1-0df9-4583-beb9-56f83fee5c0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/37f813f1-0df9-4583-beb9-56f83fee5c0a\",\n \"name\": \"37f813f1-0df9-4583-beb9-56f83fee5c0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:45.206355596Z\",\n \"endTime\": \"2026-09-28T13:22:51.172490782Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.3 is Partial (want Completed), started 26m23s ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, cluster-image-registry-operator deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"10442c188c833ffedc7ef4da292665baff0c37e673939cc22c4ae73341b8897f","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, cluster-image-registry-operator, community-operators-catalog, hosted-cluster-config-operator, kube-apiserver, openshift-apiserver, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"41ea574412391ed827695ede6ea7e3b08c5f1c51c6df393a34c7a052abb72f8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"alert","signature_id":"0eaacea327a774d3a40fa905f3b0b3aa306082f3ce30942bab1dbcdad123919a","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:06:06Z\n Ended: 2026-09-28T13:14:06Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j8288256-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", region=\"centralus\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j8288256-mgmt-2\nNamespace: ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"460d3f83b8324e3676509edd13eed8604dcc656196b0b5f1815c03043787c043","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"89f9db9301af68ce3054907127a6ba7b3d74e22731fd79fdc7a6996e89a63f9d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0004bd2a8\u003e: \n resource group \"rg-candidate-4-21-phpjd5zrm7d6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-5p8bwm\" in resourcegroup=\"rg-candidate-4-21-phpjd5zrm7d6\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8e63bd1c-cb86-4f54-9bc3-fd86e73fbb80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8e63bd1c-cb86-4f54-9bc3-fd86e73fbb80\",\n \"name\": \"8e63bd1c-cb86-4f54-9bc3-fd86e73fbb80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:20:19.409419895Z\",\n \"endTime\": \"2026-09-28T13:44:21.013354483Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41am2lgq0mrglm8sdpilfbg6dbc207 still exists (deletion dispatched at 2026-09-28T13:20:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0004bd2a8\u003e: \n resource group \"rg-candidate-4-21-phpjd5zrm7d6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-21-5p8bwm\" in resourcegroup=\"rg-candidate-4-21-phpjd5zrm7d6\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8e63bd1c-cb86-4f54-9bc3-fd86e73fbb80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8e63bd1c-cb86-4f54-9bc3-fd86e73fbb80\",\n \"name\": \"8e63bd1c-cb86-4f54-9bc3-fd86e73fbb80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:20:19.409419895Z\",\n \"endTime\": \"2026-09-28T13:44:21.013354483Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41am2lgq0mrglm8sdpilfbg6dbc207 still exists (deletion dispatched at 2026-09-28T13:20:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"46fa787293920178d8dee49d2134cf84bd1820a0280d23306c83335a19e0a5ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"alert","signature_id":"44087d2f83ae7baacb227e44ff9437079810ed6d36fa5babed386db17625fc3d","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:01:03Z\n Ended: 2026-09-28T13:36:01Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8288256-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8288256-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"5049d5412057fbfd7d177745aab4fe58b1e2760f729fcb63da9317784aa3949a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"664087fc7585f829fbe7a6c474f2c0e06ccf5f5a842a67cc4a4c2cf5f9ca990d","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea49c0\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-q4hdh7csw5qs\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/daab790a-4984-45ab-a2a3-a3f37cd79508\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/daab790a-4984-45ab-a2a3-a3f37cd79508\",\n \"name\": \"daab790a-4984-45ab-a2a3-a3f37cd79508\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:48.885600114Z\",\n \"endTime\": \"2026-09-28T13:22:50.894287651Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m10s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ea49c0\u003e: \n failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-q4hdh7csw5qs\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/daab790a-4984-45ab-a2a3-a3f37cd79508\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/daab790a-4984-45ab-a2a3-a3f37cd79508\",\n \"name\": \"daab790a-4984-45ab-a2a3-a3f37cd79508\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:48.885600114Z\",\n \"endTime\": \"2026-09-28T13:22:50.894287651Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m10s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"701e995fcfccdfe7442857d195f4901c520a122c32311a8b3a2b43413ffe1352","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"e7153cfe927d09bf871348873ff5d0a76b6b72889ee031a3643abac543c6012d","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d60180\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-txbk9bmdh8wg\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ab8bf70-a76d-451a-a68e-55fd5d0807a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ab8bf70-a76d-451a-a68e-55fd5d0807a5\",\n \"name\": \"7ab8bf70-a76d-451a-a68e-55fd5d0807a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:22.508354202Z\",\n \"endTime\": \"2026-09-28T13:22:30.229773982Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 26m20s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d60180\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-txbk9bmdh8wg\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ab8bf70-a76d-451a-a68e-55fd5d0807a5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7ab8bf70-a76d-451a-a68e-55fd5d0807a5\",\n \"name\": \"7ab8bf70-a76d-451a-a68e-55fd5d0807a5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:22.508354202Z\",\n \"endTime\": \"2026-09-28T13:22:30.229773982Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 26m20s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8336c234ad254c364931266b57449c01d70c705e10281249defaf505a41dc5a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"b51f9ec76dc47f22d30731eb4fcdfa609aef5b3aaca327c9172c808dc60d7a3c","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012643d8\u003e: \n resource group \"rg-candidate-4-20-5mk9bsj2ctvs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-rp6nvl\" in resourcegroup=\"rg-candidate-4-20-5mk9bsj2ctvs\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b24fc68-0b47-4d43-852a-f452e6006f54\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b24fc68-0b47-4d43-852a-f452e6006f54\",\n \"name\": \"7b24fc68-0b47-4d43-852a-f452e6006f54\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:14:15.967505225Z\",\n \"endTime\": \"2026-09-28T13:38:21.093544627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41ah00kund6k88mndce49mop4aje17 still exists (deletion dispatched at 2026-09-28T13:14:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012643d8\u003e: \n resource group \"rg-candidate-4-20-5mk9bsj2ctvs\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-rp6nvl\" in resourcegroup=\"rg-candidate-4-20-5mk9bsj2ctvs\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b24fc68-0b47-4d43-852a-f452e6006f54\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7b24fc68-0b47-4d43-852a-f452e6006f54\",\n \"name\": \"7b24fc68-0b47-4d43-852a-f452e6006f54\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:14:15.967505225Z\",\n \"endTime\": \"2026-09-28T13:38:21.093544627Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41ah00kund6k88mndce49mop4aje17 still exists (deletion dispatched at 2026-09-28T13:14:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a69125f42286d6c095c88e6e08d3ba245e71cfa1023d11f43120effb6713f6f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"facbe030c0f80669a32838ce5a118445436a7e98d7047b622bb21b4ac7b42c09","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e3b0e0\u003e: \n resource group \"rg-candidate-4-22-pb89drlbvrsv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-prcc26\" in resourcegroup=\"rg-candidate-4-22-pb89drlbvrsv\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/144216a6-d87d-43b0-9650-80be413ca8c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/144216a6-d87d-43b0-9650-80be413ca8c9\",\n \"name\": \"144216a6-d87d-43b0-9650-80be413ca8c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:19:57.5093726Z\",\n \"endTime\": \"2026-09-28T13:44:00.980312186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41agprs3sucbbjdath1ukr197cicdg still exists (deletion dispatched at 2026-09-28T13:20:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e3b0e0\u003e: \n resource group \"rg-candidate-4-22-pb89drlbvrsv\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-prcc26\" in resourcegroup=\"rg-candidate-4-22-pb89drlbvrsv\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/144216a6-d87d-43b0-9650-80be413ca8c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/144216a6-d87d-43b0-9650-80be413ca8c9\",\n \"name\": \"144216a6-d87d-43b0-9650-80be413ca8c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:19:57.5093726Z\",\n \"endTime\": \"2026-09-28T13:44:00.980312186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41agprs3sucbbjdath1ukr197cicdg still exists (deletion dispatched at 2026-09-28T13:20:03Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8eb7132223df04f380d9cf88d4019c25dc9f93b2b9dee24adf39adee47d4cd48","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"alert","signature_id":"2cc7d70b704c84289c5bafb769a7557267d985237cbcee107436daf38e26c9c4","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:18:05Z\n Ended: 2026-09-28T13:31:03Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.50:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8288256/providers/microsoft.monitor/accounts/services-j8288256\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-789nk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T13:18:05Z\n Ended: 2026-09-28T13:29:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"azure-cns\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.50:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8288256/providers/microsoft.monitor/accounts/services-j8288256\", namespace=\"kube-system\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-789nk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: DaemonSet kube-system/azure-cns has not finished or progressed for at least 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired"}},{"row_id":"97021cb83bf0e47dc5b9c8590bf9db3cf65c8c91d952f1179f4b8d155e609347","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"f7d62dbf65a929971d7b445f4669c5e565d9157cf38c9ff8ed6b211c77c0fe91","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-26r6nt\" with version 4.20.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106e580\u003e: \n failed to create HCP cluster cluster-zstream-4-20-26r6nt: failed waiting for cluster=\"cluster-zstream-4-20-26r6nt\" in resourcegroup=\"rg-zstream-upgrade-4-20-6bpd6fpsn6s4\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dd629ed-2a31-430a-9bfc-f3b253048b0d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dd629ed-2a31-430a-9bfc-f3b253048b0d\",\n \"name\": \"4dd629ed-2a31-430a-9bfc-f3b253048b0d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:43.832400833Z\",\n \"endTime\": \"2026-09-28T13:22:51.48967289Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-apiserver, openshift-route-controller-manager, catalog-operator, community-operators-catalog, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-storage-operator, openshift-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, ingress-operator, cluster-policy-controller, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-26r6nt\" with version 4.20.39 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106e580\u003e: \n failed to create HCP cluster cluster-zstream-4-20-26r6nt: failed waiting for cluster=\"cluster-zstream-4-20-26r6nt\" in resourcegroup=\"rg-zstream-upgrade-4-20-6bpd6fpsn6s4\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dd629ed-2a31-430a-9bfc-f3b253048b0d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/4dd629ed-2a31-430a-9bfc-f3b253048b0d\",\n \"name\": \"4dd629ed-2a31-430a-9bfc-f3b253048b0d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:43.832400833Z\",\n \"endTime\": \"2026-09-28T13:22:51.48967289Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: kube-apiserver, openshift-route-controller-manager, catalog-operator, community-operators-catalog, hosted-cluster-config-operator, olm-operator, cluster-version-operator, cluster-storage-operator, openshift-controller-manager, dns-operator, packageserver, redhat-operators-catalog, certified-operators-catalog, cluster-network-operator, cluster-node-tuning-operator, csi-snapshot-controller-operator, cluster-image-registry-operator, ingress-operator, cluster-policy-controller, redhat-marketplace-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, hosted-cluster-config-operator deployment has 1 unavailable replicas, kube-apiserver deployment has 1 unavailable replicas, olm-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas, packageserver deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"033a35a94b422f43f42fb3fd7d6be1a15babbad3631288f522120821e13fb3db","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, hosted-cluster-config-operator, kube-apiserver, olm-operator, openshift-apiserver, packageserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9b60d5cb6f006c9c46622ff36f3486a5d612d282055954d06d9431a09023965c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"alert","signature_id":"7e5bb140f45412694ed52c011bce64b635715a19e67b5f190f74b9ede9baf738","test_name":"[aro-hcp-observability] [svc] alert KubeNodeUnreachable does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:21:03Z\n Ended: 2026-09-28T13:27:01Z\n Severity: Sev3\n Labels: alertname=\"KubeNodeUnreachable\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", effect=\"NoSchedule\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.50:8080\", job=\"kube-state-metrics\", key=\"node.kubernetes.io/unreachable\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8288256/providers/microsoft.monitor/accounts/services-j8288256\", namespace=\"prometheus\", node=\"aks-userswft3-30800232-vmss000000\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-789nk\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: aks-userswft3-30800232-vmss000000 is unreachable and some workloads may be rescheduled.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","failure_pattern":"alert [svc] KubeNodeUnreachable fired","search_query":"alert [svc] KubeNodeUnreachable fired"}},{"row_id":"ae4562136b59620a2d22fb261c4b5ca27f8a31fb6a8282cdfd134b4d650173c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"601c5bd9f09b2e0f6ac00b5a321bb62c9488bb447cb61580b5be44db79b4f73d","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--gd5x92vx9cnh/cluster-hs-pre-r2s4dx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd2480\u003e: \n failed to create HCP cluster cluster-hs-pre-r2s4dx: failed waiting for cluster=\"cluster-hs-pre-r2s4dx\" in resourcegroup=\"rg-hs-pre--gd5x92vx9cnh\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/471a7bfe-fca1-453d-8b53-ca50a511f4b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/471a7bfe-fca1-453d-8b53-ca50a511f4b4\",\n \"name\": \"471a7bfe-fca1-453d-8b53-ca50a511f4b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:54.032352928Z\",\n \"endTime\": \"2026-09-28T13:23:00.490629254Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--gd5x92vx9cnh/cluster-hs-pre-r2s4dx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fd2480\u003e: \n failed to create HCP cluster cluster-hs-pre-r2s4dx: failed waiting for cluster=\"cluster-hs-pre-r2s4dx\" in resourcegroup=\"rg-hs-pre--gd5x92vx9cnh\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/471a7bfe-fca1-453d-8b53-ca50a511f4b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/471a7bfe-fca1-453d-8b53-ca50a511f4b4\",\n \"name\": \"471a7bfe-fca1-453d-8b53-ca50a511f4b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:54.032352928Z\",\n \"endTime\": \"2026-09-28T13:23:00.490629254Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 22m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"af0981bed1bc5615a6e0a4355c7f82669dcdeab3687cee85e4b104536835f029","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"19c4e7bd3a878120f8a1e6eec2558373cd830b2818377cfd788cbadd533fad7a","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002725230\u003e: \n failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-w8pbb4pwwmm7\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-w8pbb4pwwmm7, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002725230\u003e: \n failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-w8pbb4pwwmm7\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-w8pbb4pwwmm7, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2992815bad38a5769dd245becb8213e9b3a7e85882b8db6a819eb56049478a21","failure_pattern":"UpdateHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-w8pbb4pwwmm7\" to finish updating, caused by: timeout '10.000000' minutes exceeded during UpdateHCPCluster for cluster autoscaling-hcp-cluster in resource group autoscaling-cluster-w8pbb4pwwmm7, error: context deadline exceeded"}},{"row_id":"b299cfcdd5709ad82b570263cc12cceaca074c9439ec5bb018568ed13d0d525e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"a6dfca451b27dfff658fc8f1a0157dcb703810b9eb1848556cf013c0e58694f8","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e5c108\u003e: \n resource group \"v20261001preview-qdw6lx4gfk7z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-qdw6lx4gfk7z\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee12444a-4944-4551-91b1-efd928e4d331\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee12444a-4944-4551-91b1-efd928e4d331\",\n \"name\": \"ee12444a-4944-4551-91b1-efd928e4d331\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:20:48.465989676Z\",\n \"endTime\": \"2026-09-28T13:44:50.987334733Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41ah3qate5m8su42c0558vhm7cr3k1 still exists (deletion dispatched at 2026-09-28T13:20:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e5c108\u003e: \n resource group \"v20261001preview-qdw6lx4gfk7z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-qdw6lx4gfk7z\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee12444a-4944-4551-91b1-efd928e4d331\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/ee12444a-4944-4551-91b1-efd928e4d331\",\n \"name\": \"ee12444a-4944-4551-91b1-efd928e4d331\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:20:48.465989676Z\",\n \"endTime\": \"2026-09-28T13:44:50.987334733Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41ah3qate5m8su42c0558vhm7cr3k1 still exists (deletion dispatched at 2026-09-28T13:20:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bac4aec15ef3c0dd1589e47d20746f15189b8eab672b1125609cd8fa586b1984","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"alert","signature_id":"0c6477cbe2fa9f416a53e241e5f45eeea8fd5de09d670a995c600c8dd10a36a0","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 54 time(s) (38 unknown, 16 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:06:01Z\n Ended: 2026-09-28T13:29:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41bfp4l0n502nn624hggcdkif3fakm-l7r5c9r0t4i6i9i\", pod=\"router-5f8dd5cdc8-2x7hv\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41bfp4l0n502nn624hggcdkif3fakm-l7r5c9r0t4i6i9i/router-5f8dd5cdc8-2x7hv has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T13:07:01Z\n Ended: 2026-09-28T13:13:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u\", pod=\"cluster-network-operator-745b94c8d6-xgbrm\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u/cluster-network-operator-745b94c8d6-xgbrm has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-28T13:07:01Z\n Ended: 2026-09-28T13:29:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b7tbna4i6junuk36l82slrlj348n-cilium-cluster\", pod=\"router-7f5958d584-h7qql\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41b7tbna4i6junuk36l82slrlj348n-cilium-cluster/router-7f5958d584-h7qql has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-28T13:07:01Z\n Ended: 2026-09-28T13:26:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"openshift-apiserver-77dbbb84d4-g84qr\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/openshift-apiserver-77dbbb84d4-g84qr has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-28T13:07:01Z\n Ended: 2026-09-28T13:32:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41aorgo7k4e5t9d3gl5386u8qrr1ok-b1g2t3h0x4w9w8x\", pod=\"router-869d798bbd-2c6kt\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41aorgo7k4e5t9d3gl5386u8qrr1ok-b1g2t3h0x4w9w8x/router-869d798bbd-2c6kt has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-28T13:07:01Z\n Ended: 2026-09-28T13:37:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b7tbna4i6junuk36l82slrlj348n-cilium-cluster\", pod=\"router-7f5958d584-xsp4j\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41b7tbna4i6junuk36l82slrlj348n-cilium-cluster/router-7f5958d584-xsp4j has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-28T13:08:02Z\n Ended: 2026-09-28T13:35:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41bju2fkvns9a7dk4sqfvummalvm0f-t0e8p3i4e9p7l9u\", pod=\"router-6449799f6-vwm42\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41bju2fkvns9a7dk4sqfvummalvm0f-t0e8p3i4e9p7l9u/router-6449799f6-vwm42 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-28T13:08:02Z\n Ended: 2026-09-28T13:26:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s\", pod=\"ignition-server-proxy-67ffb7567d-ndltn\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s/ignition-server-proxy-67ffb7567d-ndltn has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-28T13:08:03Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41an05susenrnl0dfs40rcas06ch4c-f7z5i4t9g6o0n5p\", pod=\"ignition-server-proxy-8699545598-fvwm5\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41an05susenrnl0dfs40rcas06ch4c-f7z5i4t9g6o0n5p/ignition-server-proxy-8699545598-fvwm5 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-28T13:08:03Z\n Ended: 2026-09-28T13:26:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"kube-controller-manager-867d6bcbc8-7zqfz\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/kube-controller-manager-867d6bcbc8-7zqfz has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-28T13:16:02Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress\", pod=\"openshift-apiserver-db5f58d87-n2wk7\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress/openshift-apiserver-db5f58d87-n2wk7 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u\", pod=\"kube-apiserver-85d5846f96-gpf28\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u/kube-apiserver-85d5846f96-gpf28 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", pod=\"openshift-apiserver-85677f65c4-wrvhg\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics/openshift-apiserver-85677f65c4-wrvhg has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress\", pod=\"network-node-identity-7549b9758b-4wb6s\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress/network-node-identity-7549b9758b-4wb6s has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"openshift-apiserver-77dbbb84d4-29r9g\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/openshift-apiserver-77dbbb84d4-29r9g has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"kube-apiserver-5c98b4447b-wvxv8\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/kube-apiserver-5c98b4447b-wvxv8 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s\", pod=\"kube-apiserver-745ff9b4db-srgc7\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s/kube-apiserver-745ff9b4db-srgc7 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j\", pod=\"openshift-apiserver-84fcf4bc4d-mcmsx\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j/openshift-apiserver-84fcf4bc4d-mcmsx has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d\", pod=\"openshift-apiserver-b6b777476-w4j8h\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d/openshift-apiserver-b6b777476-w4j8h has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d\", pod=\"network-node-identity-57484c8884-trf2c\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d/network-node-identity-57484c8884-trf2c has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"openshift-apiserver-5c68654d4b-dbjjt\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/openshift-apiserver-5c68654d4b-dbjjt has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s\", pod=\"openshift-apiserver-68c54f7686-xjxd2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41b29n97p684fi4p2rqp4ah5l33p66-l4y1n8h6k2x5m2s/openshift-apiserver-68c54f7686-xjxd2 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress\", pod=\"kube-apiserver-86c59c88cb-qrxm8\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41agsvgq9e8lrm448e04up5qhj06ch-private-ingress/kube-apiserver-86c59c88cb-qrxm8 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"kube-controller-manager-867d6bcbc8-bpsb5\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/kube-controller-manager-867d6bcbc8-bpsb5 has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j\", pod=\"network-node-identity-65ff54759c-92pcp\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j/network-node-identity-65ff54759c-92pcp has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster\", pod=\"network-node-identity-594c4dccf4-lm2n6\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41ajnib77oktimcbmm0mfdc1ps458f-ea-cluster/network-node-identity-594c4dccf4-lm2n6 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", pod=\"kube-apiserver-b8d4f754f-dqwb6\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics/kube-apiserver-b8d4f754f-dqwb6 has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d\", pod=\"kube-apiserver-957c77ffc-rg8wb\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d/kube-apiserver-957c77ffc-rg8wb has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"kube-apiserver-5c9ff4f96c-n5vjw\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/kube-apiserver-5c9ff4f96c-n5vjw has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u\", pod=\"openshift-apiserver-554784946b-mjxzz\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41ba7k8pr9s4gom607lnh4dgn28ueo-g7e6w9u8n9t7m6u/openshift-apiserver-554784946b-mjxzz has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-09-28T13:17:07Z\n Ended: 2026-09-28T13:26:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j\", pod=\"kube-apiserver-65f96cf6cc-d9g85\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am7qq1p8u0u2h4skvlto6j0jvekl-p5f5c9w6j9u4m2j/kube-apiserver-65f96cf6cc-d9g85 has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-09-28T13:19:08Z\n Ended: 2026-09-28T13:26:04Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41anli60mk7obvb9qscekiln1r7nlh-z0b5k9r4h8u4l9q\", pod=\"openshift-apiserver-5f4c5b7f7c-l5gpc\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41anli60mk7obvb9qscekiln1r7nlh-z0b5k9r4h8u4l9q/openshift-apiserver-5f4c5b7f7c-l5gpc has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-09-28T13:20:08Z\n Ended: 2026-09-28T13:26:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d\", pod=\"multus-admission-controller-78665bcc7c-5f2bh\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am8gqalmfue5fmfh8v5u9ds73ppi-h6x4e1n8s3v5m1d/multus-admission-controller-78665bcc7c-5f2bh has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-09-28T13:31:01Z\n Ended: 2026-09-28T13:41:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"csi-snapshot-controller-operator-74759598d-8dhlz\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/csi-snapshot-controller-operator-74759598d-8dhlz has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-09-28T13:31:01Z\n Ended: 2026-09-28T13:41:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"cluster-storage-operator-55cc4d6675-r7fxd\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/cluster-storage-operator-55cc4d6675-r7fxd has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-09-28T13:31:01Z\n Ended: 2026-09-28T13:41:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"cluster-network-operator-5dc56d94dc-rmrjd\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/cluster-network-operator-5dc56d94dc-rmrjd has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-09-28T13:31:01Z\n Ended: 2026-09-28T13:41:02Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"dns-operator-6d45bd4dc-c5h9t\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/dns-operator-6d45bd4dc-c5h9t has been in a non-ready state for longer than 5 minutes.\nFiring 38:\n State: Resolved\n Started: 2026-09-28T13:31:01Z\n Ended: 2026-09-28T13:41:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8288256-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p\", pod=\"ingress-operator-6c4557f7b9-x9crg\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t41am6903s65jni7s8tbqnvdtrqha12-n7t6b4k4k5z4g3p/ingress-operator-6c4557f7b9-x9crg has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"be941f2d5de00e773a0eddbcfd560fd8e256fa6ee3341133a68e1750b5497326","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"6c6b92988197eb091819159a8cd6fffc7c5e8d6899de05bd4ab24749cc9fa574","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dbafa0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xh7nfd4qdxnv\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10cf3d15-2823-4e15-9280-d4bdf793742a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10cf3d15-2823-4e15-9280-d4bdf793742a\",\n \"name\": \"10cf3d15-2823-4e15-9280-d4bdf793742a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:54:54.12824002Z\",\n \"endTime\": \"2026-09-28T13:24:00.287293188Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dbafa0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-xh7nfd4qdxnv\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10cf3d15-2823-4e15-9280-d4bdf793742a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/10cf3d15-2823-4e15-9280-d4bdf793742a\",\n \"name\": \"10cf3d15-2823-4e15-9280-d4bdf793742a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:54:54.12824002Z\",\n \"endTime\": \"2026-09-28T13:24:00.287293188Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c508ff4126c464ce021bceefa97bd6e0408dfb044fe18b30a9f0f3c2a0ff45fd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"4db5797f215c217305fb770b8ae20d1fadd3857acb12459b9d26de91ec6863e5","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6bae0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5w2hj7t2fgmv\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb21162c-bce0-4144-b0c6-c225f21cfacb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb21162c-bce0-4144-b0c6-c225f21cfacb\",\n \"name\": \"fb21162c-bce0-4144-b0c6-c225f21cfacb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:32.768656716Z\",\n \"endTime\": \"2026-09-28T13:22:40.579367905Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m26s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a6bae0\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-5w2hj7t2fgmv\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb21162c-bce0-4144-b0c6-c225f21cfacb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/fb21162c-bce0-4144-b0c6-c225f21cfacb\",\n \"name\": \"fb21162c-bce0-4144-b0c6-c225f21cfacb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:53:32.768656716Z\",\n \"endTime\": \"2026-09-28T13:22:40.579367905Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 25m26s ago\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"cf84d393c91650c905f6d946de640ce2cac73f9ef227f1a0ca0a50bb05355147","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"8f4ad0dcbe474318d38b8ceb4a2664e001ad4acd5b59bb3f9f027259298ec977","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000ef8198\u003e: \n resource group \"feature-aggregation-xs2z7sms8dlg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-xs2z7sms8dlg\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54660575-548c-4595-879d-9a906af13f0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54660575-548c-4595-879d-9a906af13f0e\",\n \"name\": \"54660575-548c-4595-879d-9a906af13f0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:24:12.67709827Z\",\n \"endTime\": \"2026-09-28T13:48:20.982209436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41bq34gqtdianb4nt2b6ogejjvitge still exists (deletion dispatched at 2026-09-28T13:24:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000ef8198\u003e: \n resource group \"feature-aggregation-xs2z7sms8dlg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-xs2z7sms8dlg\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54660575-548c-4595-879d-9a906af13f0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/54660575-548c-4595-879d-9a906af13f0e\",\n \"name\": \"54660575-548c-4595-879d-9a906af13f0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:24:12.67709827Z\",\n \"endTime\": \"2026-09-28T13:48:20.982209436Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41bq34gqtdianb4nt2b6ogejjvitge still exists (deletion dispatched at 2026-09-28T13:24:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"decf356563b25cb8ecdcbd5637481a7ba9e06f56c957882f4d1e17967cb23e7d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"bf243537c8da415f24304bc3723e10ad081d488b11614cdabdb3f6c9dd66a913","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a36160\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-znb5fvxdcbts\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8be6cab6-4516-4904-95c9-f95cbd718be0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8be6cab6-4516-4904-95c9-f95cbd718be0\",\n \"name\": \"8be6cab6-4516-4904-95c9-f95cbd718be0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:55:42.029686391Z\",\n \"endTime\": \"2026-09-28T13:24:50.589882071Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m31s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a36160\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-znb5fvxdcbts\" to finish creating: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8be6cab6-4516-4904-95c9-f95cbd718be0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8be6cab6-4516-4904-95c9-f95cbd718be0\",\n \"name\": \"8be6cab6-4516-4904-95c9-f95cbd718be0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T12:55:42.029686391Z\",\n \"endTime\": \"2026-09-28T13:24:50.589882071Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m31s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e145900a20493bf88b83613a43300b8a82de4dcae8be55f37457843ec76bdaf0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"alert","signature_id":"f41df77c38855e0775eecadb32a0db47fcca98f18d194ba1e30d8c87c6bfa835","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:06:05Z\n Ended: 2026-09-28T13:14:04Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j8288256-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", region=\"centralus\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j8288256-mgmt-2\nNamespace: ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"ebda52f26b310e5cb6e180e9dd414bc37a518f6540fa67315e07a9e974eac416","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"alert","signature_id":"c42d12564755be56bc68f22fe0f502fb5d2224e2c44a2d94b8b4566c7d00cd5a","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T13:09:09Z\n Ended: 2026-09-28T13:23:06Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j8288256-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j8288256-mgmt-2\nNamespace: ocm-arohcpci01-2t41anmrr6qaria4givc77cctbvucc8g-ksm-hcp-metrics","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"f4d999b36b9b9c722bae9afd0333f1f66ff39e7e4532d1c923ccb3dbb8bde5e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104539007848288256","occurred_at":"2026-09-28T11:49:34Z","failed_at":"e2e","signature_id":"7f117a546143ac8ae4627871bef65f861c9cbed6350d628405581b5ef7dd1fa8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f043c0\u003e: \n resource group \"rg-candidate-4-23-4pqspxzwdtwh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-kgnzqx\" in resourcegroup=\"rg-candidate-4-23-4pqspxzwdtwh\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ff9462-d02e-4eaf-a164-a96ef15aebfd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ff9462-d02e-4eaf-a164-a96ef15aebfd\",\n \"name\": \"02ff9462-d02e-4eaf-a164-a96ef15aebfd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:18:50.813351207Z\",\n \"endTime\": \"2026-09-28T13:42:51.002385237Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41ao91k1fdh4d610bhaes144ec06jp still exists (deletion dispatched at 2026-09-28T13:18:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f043c0\u003e: \n resource group \"rg-candidate-4-23-4pqspxzwdtwh\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-kgnzqx\" in resourcegroup=\"rg-candidate-4-23-4pqspxzwdtwh\" to finish deleting: GET https://rp.j8288256.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ff9462-d02e-4eaf-a164-a96ef15aebfd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/02ff9462-d02e-4eaf-a164-a96ef15aebfd\",\n \"name\": \"02ff9462-d02e-4eaf-a164-a96ef15aebfd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T13:18:50.813351207Z\",\n \"endTime\": \"2026-09-28T13:42:51.002385237Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t41ao91k1fdh4d610bhaes144ec06jp still exists (deletion dispatched at 2026-09-28T13:18:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":23,"failure_pattern_ids":["033a35a94b422f43f42fb3fd7d6be1a15babbad3631288f522120821e13fb3db","10442c188c833ffedc7ef4da292665baff0c37e673939cc22c4ae73341b8897f","2992815bad38a5769dd245becb8213e9b3a7e85882b8db6a819eb56049478a21","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","519fe0fe7478bebfb7ef939c84f76fa0281886d85cb376148838a1878147b618","6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","7db32735f2411925df9899c98d2baea72687de263bdcb054c1150ac9c1a416fe","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c"],"matched_occurrences":23,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6884,"pr_state":"open","pr_sha":"50a683fcdea603a7e5aec2dbc6a88e41a26babc3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T11:10:28Z","started_at":"2026-09-28T11:10:28Z","completed_at":"2026-09-28T13:13:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T13:18:05.638472053Z","timing_metadata_checked_at":"2026-09-28T13:18:05.638472053Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T13:18:05.64229254Z","region_metadata_checked_at":"2026-09-28T13:18:05.64229254Z"},"failed_at":["alert"],"failed_test_count":5,"occurrences":[{"row_id":"13bd63e83b2f1be357a52c29430b49f0306c9a232120097254da8730920e9715","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","failed_at":"alert","signature_id":"823850876940620727554ba4d2a603b64fa3d3d9b402a7fd7add96c0cc3b5926","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:57:33Z\n Ended: 2026-09-28T12:43:33Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j0514944-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0514944-mgmt-1\nNamespace: ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"1c1ec5dc78ff975a47ac5444e457c0b18c2c76a5c19a5a62ce7dca9fa7d78363","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","failed_at":"alert","signature_id":"7630ab66e6fcc7b1b99335513790b90b490d52ddfbc4f6082579503d0e184b53","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:57:33Z\n Ended: 2026-09-28T12:43:33Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j0514944-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0514944-mgmt-1\nNamespace: ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"5cc4b408e6b4431cd454924e011bfb826dcf9c4af339164bc349036e72733e64","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","failed_at":"alert","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T12:00:53Z\n Ended: 2026-09-28T12:14:51Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0514944-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T12:06:51Z\n Ended: 2026-09-28T12:18:52Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j0514944-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"99b2a3d7f4af3de9e11b72cc8d0d263af9321b83c51023bafe9fb0fd783793ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","failed_at":"alert","signature_id":"d3bf0fab68d1955c534da4c546dfc7130a657d5faba2b998c4dfd5bfd50c00bf","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:54:31Z\n Ended: 2026-09-28T12:18:28Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j0514944-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0514944-mgmt-1\nNamespace: ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"d39c30f827397a611ba64a2d57b1f27e007be48ba0c44336443543db897cbada","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529166530514944","occurred_at":"2026-09-28T11:10:28Z","failed_at":"alert","signature_id":"4782a74e9029e3cff62109ab7b8935f9b106a03d14ad0315f92c513a27406fce","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:54:32Z\n Ended: 2026-09-28T12:18:31Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j0514944-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0514944-mgmt-1\nNamespace: ocm-arohcpci01-2t409ludea907cjj1c8fbcf54pvpn5p6-t9r3k6z8u9o6d0a","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529106451304448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7182,"pr_state":"closed","pr_sha":"b56a252408226d8ed3ec180421d2845783ace28c","final_merged_sha":"b56a252408226d8ed3ec180421d2845783ace28c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T11:10:13Z","started_at":"2026-09-28T11:10:13Z","completed_at":"2026-09-28T13:29:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T13:39:17.810802951Z","timing_metadata_checked_at":"2026-09-28T13:39:17.810802951Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T13:39:17.813471242Z","region_metadata_checked_at":"2026-09-28T13:39:17.813471242Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"cd0676cc2819038d84758fc5221a48bc0644c64e95e921e2c2540f4d3d24217b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529106451304448","occurred_at":"2026-09-28T11:10:13Z","failed_at":"alert","signature_id":"a482699276ea2332ae41de1259b09677409d1eb32ebc3d34db43a27f7addaa39","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T12:03:41Z\n Ended: 2026-09-28T12:36:41Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1304448-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1304448-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"f4600a325fc74cb6b361e9e7ace6d0adbfa754116f951e2001253c9a3a7861fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7182/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104529106451304448","occurred_at":"2026-09-28T11:10:13Z","failed_at":"alert","signature_id":"89c2e195205ab611a89f2ebc0268431d62c27ceb77950203b96368cf1d01686e","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T12:25:41Z\n Ended: 2026-09-28T12:31:44Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1304448-svc\", component=\"backend\", name=\"csstatedump\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104526841447452672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T11:00:39Z","started_at":"2026-09-28T11:00:39Z","completed_at":"2026-09-28T11:54:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T12:04:37.093184847Z","timing_metadata_checked_at":"2026-09-28T12:04:37.093184847Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T12:04:37.098958029Z","region_metadata_checked_at":"2026-09-28T12:04:37.098958029Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"cdc331d94fa401b47b14c8017a9a7cb5ca1e43dc8bfc90ab859dc20f558375fa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104526841447452672","occurred_at":"2026-09-28T11:00:39Z","failed_at":"provision","signature_id":"7ee3e6eb7ec864a3b373ec2c719dff573935d530dea03477c59c6bd3db3e8258","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=1)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T11:34:50.835Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T11:47:58.887Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=1 err=\"stamp 1: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T11:34:50.844839709Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T11:34:53.110680532Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T11:34:53.249915799Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T11:47:58.883999384Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"68FDB0D02588B7A4\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7452672-mgmt-1/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j7452672-mgmt-1/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T11:45:45.8915868Z\\\",\\\"duration\\\":\\\"PT1M54.7878497S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T11:47:58.884100077Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"2C437C987C8C8646\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7452672-mgmt-1/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j7452672-mgmt-1/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T11:45:45.8915868Z\\\",\\\"duration\\\":\\\"PT1M37.708543S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-30789729-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-30789729-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T11:47:58.8841827Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"1D4A4ACD08AFCCDC\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7452672-mgmt-1/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j7452672-mgmt-1/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T11:34:57.5557466Z\\\",\\\"duration\\\":\\\"PT12M43.6290559S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T11:47:58.88420525Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"6F7320C9245928C1\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7452672-mgmt-1/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j7452672-mgmt-1/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T11:45:45.8915868Z\\\",\\\"duration\\\":\\\"PT1M21.9184137S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T11:47:58.884223601Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"873F83FBC798DF6B\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7452672-mgmt-1/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j7452672-mgmt-1/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T11:45:45.8915868Z\\\",\\\"duration\\\":\\\"PT1M22.7299182S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T11:47:58.884239541Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"D548DA7D42E659A4\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7452672-mgmt-1\\\",\\\"resourceName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T11:34:57.5557466Z\\\",\\\"duration\\\":\\\"PT10M47.9375428S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T11:47:58.884272442Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j7452672-mgmt-1\\\",\\\"clusterName\\\":\\\"ci01-j7452672-mgmt-1\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7452672-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584110115920308752\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1/providers/Microsoft.ContainerService/managedClusters/ci01-j7452672-mgmt-1/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-30789729-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j7452672-mgmt-1-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-30789729-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"13m8.039438803s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7217/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104526339443789824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7217,"pr_state":"closed","pr_sha":"472a1ff3d5d4b88a6d4ffba2a1de439ac9966bda","final_merged_sha":"472a1ff3d5d4b88a6d4ffba2a1de439ac9966bda","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T10:59:14Z","started_at":"2026-09-28T10:59:14Z","completed_at":"2026-09-28T13:24:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T13:28:18.33744816Z","timing_metadata_checked_at":"2026-09-28T13:28:18.33744816Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T13:28:18.340383451Z","region_metadata_checked_at":"2026-09-28T13:28:18.340383451Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"a8cf8eaf778c5708a9483ac39060673e426336636a5209efa44fac6698682479","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7217/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104526339443789824","occurred_at":"2026-09-28T10:59:14Z","failed_at":"alert","signature_id":"cd68e3e4b37b20fc25bbf6e62af045b96ffb4fea1b64e10b73cc04c10aa0cbf8","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T12:26:58Z\n Ended: 2026-09-28T12:37:54Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3789824-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3789824-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7218,"pr_state":"closed","pr_sha":"ddb05e16dbc3238b583ebb3c3076ac277e0d2994","final_merged_sha":"ddb05e16dbc3238b583ebb3c3076ac277e0d2994","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T10:31:16Z","started_at":"2026-09-28T10:31:16Z","completed_at":"2026-09-28T12:43:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T12:57:19.83653599Z","timing_metadata_checked_at":"2026-09-28T12:57:19.83653599Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T12:57:19.83945238Z","region_metadata_checked_at":"2026-09-28T12:57:19.83945238Z"},"failed_at":["alert"],"failed_test_count":5,"occurrences":[{"row_id":"20335d1f491cb4ca3498d5bac9567fa4cffa0978507382f204655ee300091cdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","failed_at":"alert","signature_id":"c224092c3a3b7cc45c3855e0f2ef222d7078ecb744c1f53a5b2ec26cbbaf5fc3","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:21:38Z\n Ended: 2026-09-28T11:54:40Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7241856-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7241856-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"3e25c7ab179c579698f7d03d7ebb65c1ba310c7d985e915c7d37818365172bf5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","failed_at":"alert","signature_id":"a3308501a5c9a7128ceab2567a69a869e6a77d0446039c5c3aacec9cc2b85dba","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:44:58Z\n Ended: 2026-09-28T12:14:55Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j7241856-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j7241856-mgmt-2\nNamespace: ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"3fc0a9d05d821babe5fa3d81ba9d691e21387ea7c7ab1a35777ec0a21eedd2ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","failed_at":"alert","signature_id":"68426bd9cacc1090d6633665a7e4523175f75051187e125f21864cda11e4b81d","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:24:54Z\n Ended: 2026-09-28T11:45:56Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j7241856-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j7241856-mgmt-2\nNamespace: ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"6f8a3e28a9b91ac9a9f7a2764ae4038e026a1fb4c90d536790d9ff4fb0dfd630","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","failed_at":"alert","signature_id":"52d0cc354a3b86402491a5b57b32bfdf6698b722ffc683b472b627660b31af7d","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:27:55Z\n Ended: 2026-09-28T12:10:56Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j7241856-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j7241856-mgmt-2\nNamespace: ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"f7e1a371fc7f885ad45f72873603ffe0099d79291a93611375ac0645fc4ebfda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7218/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104519303637241856","occurred_at":"2026-09-28T10:31:16Z","failed_at":"alert","signature_id":"ad4b6e564bf43ba35f8f2b2b54565b1dcd6add304f45b0a3ec90330d94c109cc","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:41:58Z\n Ended: 2026-09-28T11:49:56Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j7241856-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j7241856-mgmt-2\nNamespace: ocm-arohcpci01-2t3vrficcj0hcpor2hrq2igqcsmsl70r-private-kas","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104517513030144000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7106,"pr_state":"closed","pr_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","final_merged_sha":"ce81d96c8a06e61b8764c6e6f0abeed76d07e74a","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T10:24:09Z","started_at":"2026-09-28T10:24:09Z","completed_at":"2026-09-28T12:37:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T12:46:54.621492344Z","timing_metadata_checked_at":"2026-09-28T12:46:54.621492344Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T12:46:54.624695834Z","region_metadata_checked_at":"2026-09-28T12:46:54.624695834Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"09850187ef516603e9216e93446b7a5d7d1a2745489ab8a9db99a33ac7687466","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104517513030144000","occurred_at":"2026-09-28T10:24:09Z","failed_at":"alert","signature_id":"3c33a5425afe4d1a10fafe1ea63cdaa957078a492eb938968ad4b5da432c09d0","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:22:20Z\n Ended: 2026-09-28T12:10:21Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j0144000-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3vp4t5eip9aogjiq8jka0asjt0oqsm-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j0144000-mgmt-2\nNamespace: ocm-arohcpci01-2t3vp4t5eip9aogjiq8jka0asjt0oqsm-private-kas","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"5416813865fd8f514181127d8a15f0291725d63281a980499982e44f8c803b82","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7106/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104517513030144000","occurred_at":"2026-09-28T10:24:09Z","failed_at":"alert","signature_id":"0402706d7b08da337e6c9a6ff7280d7bcc0c4516a43230d98d67a68ced2ef9a4","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T11:19:18Z\n Ended: 2026-09-28T11:45:22Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j0144000-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3vp4t5eip9aogjiq8jka0asjt0oqsm-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j0144000-mgmt-2\nNamespace: ocm-arohcpci01-2t3vp4t5eip9aogjiq8jka0asjt0oqsm-private-kas","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500979540955136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T09:18:22Z","started_at":"2026-09-28T09:18:22Z","completed_at":"2026-09-28T11:47:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T11:54:16.181875653Z","timing_metadata_checked_at":"2026-09-28T11:54:16.181875653Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T11:54:16.184536644Z","region_metadata_checked_at":"2026-09-28T11:54:16.184536644Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"330b8957297c86030a788b43f4d7a41cd885924f675aaba3aeed2ba94bc7ecdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500979540955136","occurred_at":"2026-09-28T09:18:22Z","failed_at":"e2e","signature_id":"4205a7cb8605e3a20406441c549ad2d3c1b449fed823b20a747976cb9404f3ee","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-htjpjq\" for cluster \"cluster-hs-pre-htjpjq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9e360\u003e: \n failed to create NodePool np-htjpjq: failed waiting for nodepool=\"np-htjpjq\" for cluster \"cluster-hs-pre-htjpjq\" in resourcegroup=\"rg-hs-pre--7q459bkmgf27\" to finish creating: GET https://rp.j0955136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1c41955-68c5-43d7-a640-907c30bcadb1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1c41955-68c5-43d7-a640-907c30bcadb1\",\n \"name\": \"d1c41955-68c5-43d7-a640-907c30bcadb1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T10:29:10.039353403Z\",\n \"endTime\": \"2026-09-28T10:48:12.965050693Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6f9w7w4s9l9e8y-np-htjpjq-hqvsc-tsmn8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--7q459bkmgf27-htjpjq--managed/providers/Microsoft.Compute/virtualMachines/u6f9w7w4s9l9e8y-np-htjpjq-hqvsc-tsmn8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6f9w7w4s9l9e8y-np-htjpjq-hqvsc-tsmn8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--7q459bkmgf27-htjpjq--managed/providers/Microsoft.Compute/virtualMachines/u6f9w7w4s9l9e8y-np-htjpjq-hqvsc-tsmn8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:112]: failed to create node pool \"np-htjpjq\" for cluster \"cluster-hs-pre-htjpjq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a9e360\u003e: \n failed to create NodePool np-htjpjq: failed waiting for nodepool=\"np-htjpjq\" for cluster \"cluster-hs-pre-htjpjq\" in resourcegroup=\"rg-hs-pre--7q459bkmgf27\" to finish creating: GET https://rp.j0955136.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1c41955-68c5-43d7-a640-907c30bcadb1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d1c41955-68c5-43d7-a640-907c30bcadb1\",\n \"name\": \"d1c41955-68c5-43d7-a640-907c30bcadb1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T10:29:10.039353403Z\",\n \"endTime\": \"2026-09-28T10:48:12.965050693Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u6f9w7w4s9l9e8y-np-htjpjq-hqvsc-tsmn8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--7q459bkmgf27-htjpjq--managed/providers/Microsoft.Compute/virtualMachines/u6f9w7w4s9l9e8y-np-htjpjq-hqvsc-tsmn8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u6f9w7w4s9l9e8y-np-htjpjq-hqvsc-tsmn8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-hs-pre--7q459bkmgf27-htjpjq--managed/providers/Microsoft.Compute/virtualMachines/u6f9w7w4s9l9e8y-np-htjpjq-hqvsc-tsmn8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"75e9ef6887d6c8999a61da9931b29b3e25c406ff85d668a2684f009b0fd17ad7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500979540955136","occurred_at":"2026-09-28T09:18:22Z","failed_at":"alert","signature_id":"8ee7b403e6fe4426552b84f141c27a733cfae0d13bff73799620b851e1e404d5","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:53:59Z\n Ended: 2026-09-28T11:00:01Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0955136-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0955136-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500977863233536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T09:18:20Z","started_at":"2026-09-28T09:18:20Z","completed_at":"2026-09-28T11:54:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T12:04:36.964654052Z","timing_metadata_checked_at":"2026-09-28T12:04:36.964654052Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T12:04:36.967544343Z","region_metadata_checked_at":"2026-09-28T12:04:36.967544343Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"062a9729760382f7f594585bca7b522b5a0b05a02a504cbbd46b347f7db539e5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500977863233536","occurred_at":"2026-09-28T09:18:20Z","failed_at":"alert","signature_id":"5db35e658cd311f66e3a715283d477fc14e8743c4f890d427402e8cb74dccf70","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:40:21Z\n Ended: 2026-09-28T11:23:22Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j3233536-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3v2u1i7qsc0v7mcq5u2f0jotsatk8u-d2u4p8t3s9k5z2b\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j3233536-mgmt-2\nNamespace: ocm-arohcpci01-2t3v2u1i7qsc0v7mcq5u2f0jotsatk8u-d2u4p8t3s9k5z2b","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"a042c21a73d4bc78921192ac2a08724a2f8f516f705c080b7da881c6f739427f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500977863233536","occurred_at":"2026-09-28T09:18:20Z","failed_at":"alert","signature_id":"adc03018ca13b9a200d0b335ac06a5da309ea6695cbde1885be48929840e5142","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:32:27Z\n Ended: 2026-09-28T10:58:22Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j3233536-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3v2u1i7qsc0v7mcq5u2f0jotsatk8u-d2u4p8t3s9k5z2b\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j3233536-mgmt-2\nNamespace: ocm-arohcpci01-2t3v2u1i7qsc0v7mcq5u2f0jotsatk8u-d2u4p8t3s9k5z2b","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"d5bc1b38091c07d0431ca8f6d26892119d280d03c320ff748ddae13f9e48a75b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500977863233536","occurred_at":"2026-09-28T09:18:20Z","failed_at":"alert","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:36:45Z\n Ended: 2026-09-28T10:54:46Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3233536-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T10:36:45Z\n Ended: 2026-09-28T10:57:52Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3233536-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500975346651136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"closed","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T09:18:18Z","started_at":"2026-09-28T09:18:18Z","completed_at":"2026-09-28T09:37:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T09:47:55.107010576Z","timing_metadata_checked_at":"2026-09-28T09:47:55.107010576Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-28T09:47:55.109815767Z","region_metadata_checked_at":"2026-09-28T10:50:42.150792964Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d6a2987824c84fa4b391036ae1817b5c0d937d3e8c6b557f1b1e28e6e86595dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500975346651136","occurred_at":"2026-09-28T09:18:18Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500972830068736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T09:18:16Z","started_at":"2026-09-28T09:18:16Z","completed_at":"2026-09-28T11:45:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T11:54:15.872457233Z","timing_metadata_checked_at":"2026-09-28T11:54:15.872457233Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T11:54:15.876074721Z","region_metadata_checked_at":"2026-09-28T11:54:15.876074721Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"cab31e941ef2319edcbebb9f328696d4f25649c67deec2f9886cbadf67aedcb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500972830068736","occurred_at":"2026-09-28T09:18:16Z","failed_at":"alert","signature_id":"0c3acf11461e0fac9b5520d458eb6f01cdf27f718bc25ff896d1d3bf8a9936d8","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:52:35Z\n Ended: 2026-09-28T11:00:37Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0068736-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0068736-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500902567088128","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7202,"pr_state":"open","pr_sha":"bf798191f1e4784ba8c222bbae084b193940b0ff","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T09:18:09Z","started_at":"2026-09-28T09:18:09Z","completed_at":"2026-09-28T11:49:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T11:54:16.367104166Z","timing_metadata_checked_at":"2026-09-28T11:54:16.367104166Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T11:54:16.370003157Z","region_metadata_checked_at":"2026-09-28T11:54:16.370003157Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"9101067dee2b86629f178e9afde4b84cc435207f702d382e2804e0adfebbd88b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500902567088128","occurred_at":"2026-09-28T09:18:09Z","failed_at":"alert","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:33:36Z\n Ended: 2026-09-28T10:45:36Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j7088128-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"a30a0c5a58dd0d6c129470aaae57714c147ad6c335e42cce9fe7219bedb663e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500902567088128","occurred_at":"2026-09-28T09:18:09Z","failed_at":"alert","signature_id":"5c7d7ddfa07a3a6a4e8abf4f2a8fa726968d0d86dcd83659570e6e8d5c578e36","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:53:56Z\n Ended: 2026-09-28T11:00:56Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7088128-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7088128-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500868232515584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7203,"pr_state":"open","pr_sha":"75d72389678ef3781f99923a095f0f5dfe0094f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T09:18:01Z","started_at":"2026-09-28T09:18:01Z","completed_at":"2026-09-28T11:38:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T11:43:17.532484661Z","timing_metadata_checked_at":"2026-09-28T11:43:17.532484661Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T11:43:17.535022653Z","region_metadata_checked_at":"2026-09-28T11:43:17.535022653Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"052ec9a4f6a19ca484d046deb78470453ef9f1eea1b4046e44f66d3e63e4931c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500868232515584","occurred_at":"2026-09-28T09:18:01Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"3a6d5638bc4382d666cb823fec826a71ebbfdde24b3febf1b6f16ac50adc7be9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500868232515584","occurred_at":"2026-09-28T09:18:01Z","failed_at":"e2e","signature_id":"af5dba612f5bcafa125e14121eae54d6911f7b73c83a90c9caccd95170639937","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138a980\u003e: \n failed to create HCP cluster np-dg-minor-w2lmrh: failed waiting for cluster=\"np-dg-minor-w2lmrh\" in resourcegroup=\"rg-np-dg-minor-w2lmrh-zzht7hmgmswz\" to finish creating: GET https://rp.j2515584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\",\n \"name\": \"8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T10:35:16.490368834Z\",\n \"endTime\": \"2026-09-28T11:04:17.550861433Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 1m2s ago; version 4.22.16 is Partial (want Completed), started 7m32s ago; version 4.22.16 is Partial (want Completed), started 11m2s ago; version 4.22.16 is Partial (want Completed), started 18m25s ago; version 4.22.16 is Partial (want Completed), started 20m51s ago; version 4.22.16 is Partial (want Completed), started 23m2s ago; version 4.22.16 is Partial (want Completed), started 26m19s ago; version 4.22.16 is Partial (want Completed), started 27m32s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:722]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00138a980\u003e: \n failed to create HCP cluster np-dg-minor-w2lmrh: failed waiting for cluster=\"np-dg-minor-w2lmrh\" in resourcegroup=\"rg-np-dg-minor-w2lmrh-zzht7hmgmswz\" to finish creating: GET https://rp.j2515584.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\",\n \"name\": \"8a4adc4d-0ad3-406d-a9bc-9c1ae046c034\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T10:35:16.490368834Z\",\n \"endTime\": \"2026-09-28T11:04:17.550861433Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.16 is Partial (want Completed), started 1m2s ago; version 4.22.16 is Partial (want Completed), started 7m32s ago; version 4.22.16 is Partial (want Completed), started 11m2s ago; version 4.22.16 is Partial (want Completed), started 18m25s ago; version 4.22.16 is Partial (want Completed), started 20m51s ago; version 4.22.16 is Partial (want Completed), started 23m2s ago; version 4.22.16 is Partial (want Completed), started 26m19s ago; version 4.22.16 is Partial (want Completed), started 27m32s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"50d71e86102fe885def0430fc65ae5eda7365e37815a984b65ed42e4e299b068","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"526a542fe86c7efd58758ac78b2b994b6350e0e20dc74ce35539b726ea7d49a6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500868232515584","occurred_at":"2026-09-28T09:18:01Z","failed_at":"alert","signature_id":"8ed8ccf041f97eca095f371db1f62c00c01ed1a5f3e20b41eb9a02f2e97c9851","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:17:06Z\n Ended: 2026-09-28T10:51:06Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2515584-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2515584-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["50d71e86102fe885def0430fc65ae5eda7365e37815a984b65ed42e4e299b068","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500002637221888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7170,"pr_state":"closed","pr_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","final_merged_sha":"715e1ab64227414bccc3c04d703bb2672c050aea","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T09:14:19Z","started_at":"2026-09-28T09:14:19Z","completed_at":"2026-09-28T11:33:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T11:43:17.404094964Z","timing_metadata_checked_at":"2026-09-28T11:43:17.404094964Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T11:43:17.407245954Z","region_metadata_checked_at":"2026-09-28T11:43:17.407245954Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"3a280bf8d4f1e1f9d2bf5ca4694d88c91b67164b67638fb0854edb781c8c8086","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7170/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104500002637221888","occurred_at":"2026-09-28T09:14:19Z","failed_at":"e2e","signature_id":"620340dd365edadda3dbd85304c7c7cde65b3e17f1f0272584c0a1bf12bb3b26","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.008s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001488a20\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:46491-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:357]: Timed out after 300.008s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc001488a20\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:46491-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104493553664135168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6884,"pr_state":"open","pr_sha":"50a683fcdea603a7e5aec2dbc6a88e41a26babc3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T08:48:57Z","started_at":"2026-09-28T08:48:57Z","completed_at":"2026-09-28T10:59:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T11:01:38.174143032Z","timing_metadata_checked_at":"2026-09-28T11:01:38.174143032Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T11:01:38.179025515Z","region_metadata_checked_at":"2026-09-28T11:01:38.179025515Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"08241144ced4c75c4e336ca39de59121a985fb083be9e91864d1d7aeb2b68380","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104493553664135168","occurred_at":"2026-09-28T08:48:57Z","failed_at":"alert","signature_id":"da5e7d6776d3b79243736efc00841e62ebccdd525b1a468588dd9309520d23c3","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T09:47:15Z\n Ended: 2026-09-28T10:31:14Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4135168-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3ucelqfhl68luucdn21goolqq53uj5-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4135168-mgmt-1\nNamespace: ocm-arohcpci01-2t3ucelqfhl68luucdn21goolqq53uj5-private-kas\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T09:49:15Z\n Ended: 2026-09-28T10:04:14Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4135168-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3uchfsr0akt7a7mrha2rd3io4p1b79-g6g2o0e4x6a3e2d\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4135168-mgmt-2\nNamespace: ocm-arohcpci01-2t3uchfsr0akt7a7mrha2rd3io4p1b79-g6g2o0e4x6a3e2d","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"61fb62eacf303373a4926de1a70729a4e5687ff46320920084a50af15d77ea99","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104493553664135168","occurred_at":"2026-09-28T08:48:57Z","failed_at":"alert","signature_id":"51b2e006e9652fd5900cb41a77a7d70291a4bee2a682adb677a4f1b150bc2599","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T09:44:14Z\n Ended: 2026-09-28T10:06:14Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4135168-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3ucelqfhl68luucdn21goolqq53uj5-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4135168-mgmt-1\nNamespace: ocm-arohcpci01-2t3ucelqfhl68luucdn21goolqq53uj5-private-kas\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T09:46:17Z\n Ended: 2026-09-28T10:02:11Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4135168-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3uchfsr0akt7a7mrha2rd3io4p1b79-g6g2o0e4x6a3e2d\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4135168-mgmt-2\nNamespace: ocm-arohcpci01-2t3uchfsr0akt7a7mrha2rd3io4p1b79-g6g2o0e4x6a3e2d","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104492976943140864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7169,"pr_state":"closed","pr_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","final_merged_sha":"efbb26e600b7d900489923ef1c5552f84ef9efb3","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T08:46:39Z","started_at":"2026-09-28T08:46:39Z","completed_at":"2026-09-28T11:00:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T11:11:50.263242523Z","timing_metadata_checked_at":"2026-09-28T11:11:50.263242523Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T11:11:50.266597812Z","region_metadata_checked_at":"2026-09-28T11:11:50.266597812Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"30e120a264e436d12808fb53df513be5f368de8c8d6db7ae7c3d58fdb1c0a18a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7169/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104492976943140864","occurred_at":"2026-09-28T08:46:39Z","failed_at":"e2e","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7217/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104491225984798720","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7217,"pr_state":"closed","pr_sha":"472a1ff3d5d4b88a6d4ffba2a1de439ac9966bda","final_merged_sha":"472a1ff3d5d4b88a6d4ffba2a1de439ac9966bda","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T08:39:42Z","started_at":"2026-09-28T08:39:42Z","completed_at":"2026-09-28T09:13:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T09:15:55.128293763Z","timing_metadata_checked_at":"2026-09-28T09:15:55.128293763Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T09:15:55.132465851Z","region_metadata_checked_at":"2026-09-28T09:15:55.132465851Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"6be68bbc640a3e45e49fede238804e3488227dd7d7b0f4cab9eb0b5cd4061ca5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7217/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104491225984798720","occurred_at":"2026-09-28T08:39:42Z","failed_at":"provision","signature_id":"1ee68ae544fafb7bb55d51a213ba66048f990c8e725c3242ac72ba56bbd6192c","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","test_suite":"templatize-pipeline","failure_text":"time=2026-09-28T09:01:02.051Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n Kind: Shell\\n Command: ./aks-cluster-create\\n\"\ntime=2026-09-28T09:08:09.651Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-28T09:01:02.061522961Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\"}\\n{\\\"time\\\":\\\"2026-09-28T09:01:04.147818707Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-28T09:01:04.440466024Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T09:08:09.64760036Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"9614EBA97D5F9800\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4798720-mgmt-2/agentPools/userswft3\\\",\\\"resourceName\\\":\\\"ci01-j4798720-mgmt-2/userswft3\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Failed\\\",\\\"timestamp\\\":\\\"2026-09-28T09:05:41.381016Z\\\",\\\"duration\\\":\\\"PT1M37.2773647S\\\",\\\"statusCode\\\":\\\"Conflict\\\",\\\"statusMessage\\\":{\\\"error\\\":{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"details\\\":[{\\\"code\\\":\\\"OverconstrainedZonalAllocationRequest\\\",\\\"message\\\":\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\",\\\"target\\\":\\\"0\\\"}],\\\"message\\\":\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-16665582-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-16665582-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\n - Availability Zone\\\\n - Differencing (Ephemeral) Disks\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\n - VM Size\\\\n\\\"},\\\"status\\\":\\\"Failed\\\"}}\\n{\\\"time\\\":\\\"2026-09-28T09:08:09.647751843Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"66FF17855A3FEDFC\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4798720-mgmt-2/agentPools/userswft2\\\",\\\"resourceName\\\":\\\"ci01-j4798720-mgmt-2/userswft2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T09:05:41.381016Z\\\",\\\"duration\\\":\\\"PT2M25.6050717S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T09:08:09.647775174Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"160C685E1A59B997\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4798720-mgmt-2/agentPools/infra1\\\",\\\"resourceName\\\":\\\"ci01-j4798720-mgmt-2/infra1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T09:01:08.8659112Z\\\",\\\"duration\\\":\\\"PT5M55.8315068S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T09:08:09.647794924Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"B1467C06696E9BAC\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4798720-mgmt-2/agentPools/infra2\\\",\\\"resourceName\\\":\\\"ci01-j4798720-mgmt-2/infra2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T09:05:41.381016Z\\\",\\\"duration\\\":\\\"PT1M22.0799243S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T09:08:09.647811774Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"50E19B7C817D66F4\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4798720-mgmt-2/agentPools/userswft1\\\",\\\"resourceName\\\":\\\"ci01-j4798720-mgmt-2/userswft1\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters/agentPools\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T09:05:41.381016Z\\\",\\\"duration\\\":\\\"PT2M24.6050443S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T09:08:09.647834635Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deployment operation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"operationID\\\":\\\"B0D1165F6FF7B302\\\",\\\"resource\\\":{\\\"id\\\":\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4798720-mgmt-2\\\",\\\"resourceName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"resourceType\\\":\\\"Microsoft.ContainerService/managedClusters\\\"},\\\"provisioningOperation\\\":\\\"Create\\\",\\\"provisioningState\\\":\\\"Succeeded\\\",\\\"timestamp\\\":\\\"2026-09-28T09:01:08.8659112Z\\\",\\\"duration\\\":\\\"PT4M32.2580835S\\\",\\\"statusCode\\\":\\\"OK\\\",\\\"statusMessage\\\":null}\\n{\\\"time\\\":\\\"2026-09-28T09:08:09.647869416Z\\\",\\\"level\\\":\\\"ERROR\\\",\\\"msg\\\":\\\"run failed\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j4798720-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j4798720-mgmt-2\\\",\\\"region\\\":\\\"centralus\\\",\\\"err\\\":\\\"waiting for deployment aks-cluster-create: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4798720-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create/operationStatuses/08584110208208621976\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 200: 200 OK\\\\nERROR CODE: DeploymentFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-create\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2/providers/Microsoft.ContainerService/managedClusters/ci01-j4798720-mgmt-2/agentPools/userswft3\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed on resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-16665582-vmss'. Please try alternative sizes 'Standard_D16s_v6' (zones 2, 1), 'Standard_D16ds_v7' (zones 2), 'Standard_D16ads_v7' (zones 1, 3, 2) instead for higher likelihood of success. Read more about improving likelihood of operation success at https://aka.ms/AKSallocation. Create or update VMSS /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4798720-mgmt-2-aks1/providers/Microsoft.Compute/virtualMachineScaleSets/aks-userswft3-16665582-vmss failed. Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\",\\\\n \\\\\\\"details\\\\\\\": [\\\\n {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"OverconstrainedZonalAllocationRequest\\\\\\\",\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"0\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\\\\\n - Availability Zone\\\\\\\\n - Differencing (Ephemeral) Disks\\\\\\\\n - Networking Constraints (such as Accelerated Networking or IPv6)\\\\\\\\n - VM Size\\\\\\\\n\\\\\\\"\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n ]\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\",\\\"elapsed\\\":\\\"7m7.586350686s\\\"}\\n exit status 1\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350","failure_pattern":"ERROR CODE: DeploymentFailed; detail code OverconstrainedZonalAllocationRequest; detail message Allocation failed. VM(s) with the following constraints cannot be allocated, because the condition is too restrictive. Please remove some constraints and try again. Constraints applied are:\\\\ - Availability Zone\\\\ - Differencing (Ephemeral) Disks\\\\ - Networking Constraints (such as Accelerated Networking or IPv6)\\\\ - VM Size\\\\; provider Microsoft.Compute","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["0ff9b1b442304fb2b24a4d531baa2405490c8caaa9c8d65d157de35512cfa350"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6900,"pr_state":"closed","pr_sha":"e7500a3658c1692b6c32705cb7edf403b704aae4","final_merged_sha":"d63de0c62ddb9d7d7de369b28c8d93807d13672c","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T08:18:43Z","started_at":"2026-09-28T08:18:43Z","completed_at":"2026-09-28T10:51:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T11:01:38.375520547Z","timing_metadata_checked_at":"2026-09-28T11:01:38.375520547Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T11:01:38.378835936Z","region_metadata_checked_at":"2026-09-28T11:01:38.378835936Z"},"failed_at":["alert","e2e"],"failed_test_count":41,"occurrences":[{"row_id":"0f4843e567de05a75e389806576613ac442cbe719ea9673df44bc99858205a6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"3e888de17147e7874eadf6ced8362014dc85d0b720dae2bc042d4c0447b893a8","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00172e090\u003e: \n resource group \"complex-cilium-kv-k9knbpmhbg7v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-k9knbpmhbg7v\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a3544af-5c77-496b-b18a-8cc097e8a79b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a3544af-5c77-496b-b18a-8cc097e8a79b\",\n \"name\": \"4a3544af-5c77-496b-b18a-8cc097e8a79b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:43.521207454Z\",\n \"endTime\": \"2026-09-28T09:54:45.223507829Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u15umjtvbfn7or4kh25ftp6mt8ubm still exists (deletion dispatched at 2026-09-28T09:30:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00172e090\u003e: \n resource group \"complex-cilium-kv-k9knbpmhbg7v\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-k9knbpmhbg7v\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a3544af-5c77-496b-b18a-8cc097e8a79b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4a3544af-5c77-496b-b18a-8cc097e8a79b\",\n \"name\": \"4a3544af-5c77-496b-b18a-8cc097e8a79b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:43.521207454Z\",\n \"endTime\": \"2026-09-28T09:54:45.223507829Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u15umjtvbfn7or4kh25ftp6mt8ubm still exists (deletion dispatched at 2026-09-28T09:30:45Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"159ee376d56c8c9e3f3865150fb5f55d547c67475f4cad4d7e50c1c5e1ac45cf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"948424a2d7453ea605ce66e874ab718d017748efea90a195d896e62ef8689abb","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c06120\u003e: \n resource group \"rg-np-version-upgrade-8l6tr2-428422fmlt4m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-8l6tr2\" in resourcegroup=\"rg-np-version-upgrade-8l6tr2-428422fmlt4m\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9bc1619-e6b7-4915-a5af-afeb34d9739c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9bc1619-e6b7-4915-a5af-afeb34d9739c\",\n \"name\": \"f9bc1619-e6b7-4915-a5af-afeb34d9739c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:47:50.839437919Z\",\n \"endTime\": \"2026-09-28T10:11:59.725052507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0lq4iolgmmgdclkhq6jg6ede2jbm still exists (deletion dispatched at 2026-09-28T09:47:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001c06120\u003e: \n resource group \"rg-np-version-upgrade-8l6tr2-428422fmlt4m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-8l6tr2\" in resourcegroup=\"rg-np-version-upgrade-8l6tr2-428422fmlt4m\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9bc1619-e6b7-4915-a5af-afeb34d9739c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9bc1619-e6b7-4915-a5af-afeb34d9739c\",\n \"name\": \"f9bc1619-e6b7-4915-a5af-afeb34d9739c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:47:50.839437919Z\",\n \"endTime\": \"2026-09-28T10:11:59.725052507Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0lq4iolgmmgdclkhq6jg6ede2jbm still exists (deletion dispatched at 2026-09-28T09:47:52Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1845d166865f27a43abe6e6b0518d1c958831aeac3687b6706f8e70b40896b1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"538ab841d7da92f151f053745afb8b6c8b0b5a9b56e4c3e14a219af6b04685ad","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001010288\u003e: \n resource group \"clusternp128-cr6m72hbhthl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-cr6m72hbhthl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2807e39f-5609-4e8d-9b63-6e0fb9dffa47\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2807e39f-5609-4e8d-9b63-6e0fb9dffa47\",\n \"name\": \"2807e39f-5609-4e8d-9b63-6e0fb9dffa47\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:26:14.153004619Z\",\n \"endTime\": \"2026-09-28T09:50:15.107483Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0n0ep16n5al4fc1ggfac6imb350l still exists (deletion dispatched at 2026-09-28T09:26:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001010288\u003e: \n resource group \"clusternp128-cr6m72hbhthl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-cr6m72hbhthl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2807e39f-5609-4e8d-9b63-6e0fb9dffa47\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2807e39f-5609-4e8d-9b63-6e0fb9dffa47\",\n \"name\": \"2807e39f-5609-4e8d-9b63-6e0fb9dffa47\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:26:14.153004619Z\",\n \"endTime\": \"2026-09-28T09:50:15.107483Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0n0ep16n5al4fc1ggfac6imb350l still exists (deletion dispatched at 2026-09-28T09:26:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"1d23d57678d658590f810d4c68c8ef893fd8a1a439b8101bb654bf13a5c300df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"655632efe6eb665737f2fe79dbe081860714186ed63d5e7bbc7561be417fe6c4","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a50120\u003e: \n resource group \"nodepool-delete-dfzrjzvh57lj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dfzrjzvh57lj\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87e2a749-adb1-4e6e-9c80-163809fbf472\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87e2a749-adb1-4e6e-9c80-163809fbf472\",\n \"name\": \"87e2a749-adb1-4e6e-9c80-163809fbf472\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:40:50.481180787Z\",\n \"endTime\": \"2026-09-28T10:04:59.753897901Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ioqrfjsn6acu7pjpg12lkcddspk still exists (deletion dispatched at 2026-09-28T09:40:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a50120\u003e: \n resource group \"nodepool-delete-dfzrjzvh57lj\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dfzrjzvh57lj\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87e2a749-adb1-4e6e-9c80-163809fbf472\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/87e2a749-adb1-4e6e-9c80-163809fbf472\",\n \"name\": \"87e2a749-adb1-4e6e-9c80-163809fbf472\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:40:50.481180787Z\",\n \"endTime\": \"2026-09-28T10:04:59.753897901Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ioqrfjsn6acu7pjpg12lkcddspk still exists (deletion dispatched at 2026-09-28T09:40:53Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"21dafecaf185cbace465ba49b8c672f76340d30103c61d7bdf1979a8b91c31e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"0fb3311ad2f9df382d87b904ce13084a4e580a3bdcc3cf253c032a2d12add8d1","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f12198\u003e: \n resource group \"feature-aggregation-qfgsqjtw55cd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qfgsqjtw55cd\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3913a806-5810-4793-abd7-5ef178b7415c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3913a806-5810-4793-abd7-5ef178b7415c\",\n \"name\": \"3913a806-5810-4793-abd7-5ef178b7415c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:37:58.390921917Z\",\n \"endTime\": \"2026-09-28T10:02:05.045669677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u1jrr7vnhpej35p0tpr9a3np9a5ic still exists (deletion dispatched at 2026-09-28T09:38:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f12198\u003e: \n resource group \"feature-aggregation-qfgsqjtw55cd\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-qfgsqjtw55cd\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3913a806-5810-4793-abd7-5ef178b7415c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3913a806-5810-4793-abd7-5ef178b7415c\",\n \"name\": \"3913a806-5810-4793-abd7-5ef178b7415c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:37:58.390921917Z\",\n \"endTime\": \"2026-09-28T10:02:05.045669677Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u1jrr7vnhpej35p0tpr9a3np9a5ic still exists (deletion dispatched at 2026-09-28T09:38:00Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"25f7f45b4fac5eeddd60bf67f5e5b9d995e73e58b0746bdf2afd4ea1c6392fa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"e6ee12a3586a43226a4838a0632196b6f3dcfaa9c97b4d56737aff26dc114226","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001722090\u003e: \n resource group \"rg-zstream-upgrade-4-20-r5d6pqgs2lcz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-xfrrpx\" in resourcegroup=\"rg-zstream-upgrade-4-20-r5d6pqgs2lcz\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/142a98c7-5abe-44fb-90f2-c5de9bb7a00e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/142a98c7-5abe-44fb-90f2-c5de9bb7a00e\",\n \"name\": \"142a98c7-5abe-44fb-90f2-c5de9bb7a00e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:23.402647295Z\",\n \"endTime\": \"2026-09-28T10:00:25.321453256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ffhfsef977jtdd70rv0ev42cvrg still exists (deletion dispatched at 2026-09-28T09:36:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001722090\u003e: \n resource group \"rg-zstream-upgrade-4-20-r5d6pqgs2lcz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-xfrrpx\" in resourcegroup=\"rg-zstream-upgrade-4-20-r5d6pqgs2lcz\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/142a98c7-5abe-44fb-90f2-c5de9bb7a00e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/142a98c7-5abe-44fb-90f2-c5de9bb7a00e\",\n \"name\": \"142a98c7-5abe-44fb-90f2-c5de9bb7a00e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:23.402647295Z\",\n \"endTime\": \"2026-09-28T10:00:25.321453256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ffhfsef977jtdd70rv0ev42cvrg still exists (deletion dispatched at 2026-09-28T09:36:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2b9f26ca6c6e11f28c0a73852ee9663dd558ff34bc0ce7cfdc714009f3b07ac1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"alert","signature_id":"c5c86d4d0c9a75c7f010a1f205acf66c6c18105fc74571e1cc5f8325e06c6ded","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T09:43:43Z\n Ended: 2026-09-28T09:49:43Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1041152-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1041152-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"2cbb7c736b415245b36314e8d4bf4895d8af160becb45b8944b88ad7dec09072","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"67aee04b8869e7e7576385277fc0bb97f3b3eb5af8dd02fb45f04ec0a2550a5e","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001178498\u003e: \n resource group \"rg-np-skip-minor-dw8b66-7pzf52lvmpf4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-dw8b66\" in resourcegroup=\"rg-np-skip-minor-dw8b66-7pzf52lvmpf4\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/55b08ad7-ac72-45f5-9b5d-12339524531b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/55b08ad7-ac72-45f5-9b5d-12339524531b\",\n \"name\": \"55b08ad7-ac72-45f5-9b5d-12339524531b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:46:43.12833531Z\",\n \"endTime\": \"2026-09-28T10:10:49.874421326Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0lvk9h4cv1mk6uo8iiei2pbj0pmg still exists (deletion dispatched at 2026-09-28T09:46:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001178498\u003e: \n resource group \"rg-np-skip-minor-dw8b66-7pzf52lvmpf4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-skip-minor-dw8b66\" in resourcegroup=\"rg-np-skip-minor-dw8b66-7pzf52lvmpf4\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/55b08ad7-ac72-45f5-9b5d-12339524531b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/55b08ad7-ac72-45f5-9b5d-12339524531b\",\n \"name\": \"55b08ad7-ac72-45f5-9b5d-12339524531b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:46:43.12833531Z\",\n \"endTime\": \"2026-09-28T10:10:49.874421326Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0lvk9h4cv1mk6uo8iiei2pbj0pmg still exists (deletion dispatched at 2026-09-28T09:46:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"35b4139de160322d9f01d0eeb12bf75f456518a48ddef001f029134b4244e4df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"5ebf9b2f69debdd5fcda540a2cf54489bef4a09bbbd816074578798dfb183457","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000bc80c0\u003e: \n resource group \"rg-cp-ystream-upgrade-4-23-wzvvqkwwvxlw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-2zhgtf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-wzvvqkwwvxlw\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d382c657-1215-4712-a751-4d1a4f16b05f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d382c657-1215-4712-a751-4d1a4f16b05f\",\n \"name\": \"d382c657-1215-4712-a751-4d1a4f16b05f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:53.979369262Z\",\n \"endTime\": \"2026-09-28T10:00:55.003547453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ic75rjvskjgforinth6afhbjsc5 still exists (deletion dispatched at 2026-09-28T09:36:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000bc80c0\u003e: \n resource group \"rg-cp-ystream-upgrade-4-23-wzvvqkwwvxlw\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-23-2zhgtf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-wzvvqkwwvxlw\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d382c657-1215-4712-a751-4d1a4f16b05f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d382c657-1215-4712-a751-4d1a4f16b05f\",\n \"name\": \"d382c657-1215-4712-a751-4d1a4f16b05f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:53.979369262Z\",\n \"endTime\": \"2026-09-28T10:00:55.003547453Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ic75rjvskjgforinth6afhbjsc5 still exists (deletion dispatched at 2026-09-28T09:36:56Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3a40f191c45b2e8fd69514d117ef84815220c8750e319d083b0a3344c69ee572","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"05edd80a677f5d832c1ef05d940180d37fe21a3734cf35f46966ff517c506b0a","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000898240\u003e: \n resource group \"oidc-wi-6h5hr74nbxbl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-6h5hr74nbxbl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de033d5e-0c9a-4b34-a70f-e0babef6ad67\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de033d5e-0c9a-4b34-a70f-e0babef6ad67\",\n \"name\": \"de033d5e-0c9a-4b34-a70f-e0babef6ad67\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:48:11.942517946Z\",\n \"endTime\": \"2026-09-28T10:12:19.666093496Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3ubc77sob0k75ihn39813gvminerda still exists (deletion dispatched at 2026-09-28T09:48:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000898240\u003e: \n resource group \"oidc-wi-6h5hr74nbxbl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-6h5hr74nbxbl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de033d5e-0c9a-4b34-a70f-e0babef6ad67\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/de033d5e-0c9a-4b34-a70f-e0babef6ad67\",\n \"name\": \"de033d5e-0c9a-4b34-a70f-e0babef6ad67\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:48:11.942517946Z\",\n \"endTime\": \"2026-09-28T10:12:19.666093496Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3ubc77sob0k75ihn39813gvminerda still exists (deletion dispatched at 2026-09-28T09:48:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3ccff6b5b15436fad25d35bc6cade0b4cc1b4b8d2234df3979e5ebd492ca8676","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"alert","signature_id":"ef765ffa110519699ce0b4ae4b0309fac4eb0cda12a23778343f5d81fb18eed6","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-28T10:12:48Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z8g6d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-28T10:15:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2d7tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-28T10:15:54Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2d7tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-28T10:16:53Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-2d7tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-28T10:17:49Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cloud-network-config-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z8g6d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/cloud-network-config-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"4019be49a0aabe1828da425458680064df6e94500466471341f7ab2b19d716d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"c565e2df29978beff1205caf7aa2ba4d94ee0ef7e6719c56612ee152f0096b19","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f7e360\u003e: \n resource group \"v20260901preview-2tskgvphk5pz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-2tskgvphk5pz\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13b9b932-13e2-4612-b6b0-df57f3cc7eb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13b9b932-13e2-4612-b6b0-df57f3cc7eb8\",\n \"name\": \"13b9b932-13e2-4612-b6b0-df57f3cc7eb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:31:10.812279232Z\",\n \"endTime\": \"2026-09-28T09:55:14.912402863Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0mu3m9kv9273jhipsv94emihr3l8 still exists (deletion dispatched at 2026-09-28T09:31:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f7e360\u003e: \n resource group \"v20260901preview-2tskgvphk5pz\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20260901\" in resourcegroup=\"v20260901preview-2tskgvphk5pz\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13b9b932-13e2-4612-b6b0-df57f3cc7eb8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/13b9b932-13e2-4612-b6b0-df57f3cc7eb8\",\n \"name\": \"13b9b932-13e2-4612-b6b0-df57f3cc7eb8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:31:10.812279232Z\",\n \"endTime\": \"2026-09-28T09:55:14.912402863Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0mu3m9kv9273jhipsv94emihr3l8 still exists (deletion dispatched at 2026-09-28T09:31:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"43a6b8efe553f98bf11c15ef26ee03812a8a26ee996f890371a9f9de1f8ff4ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"228b55142d7421c5ea9202dfd88b3f5dbcf091d3cea1ff9bcb5da8cdf2018bba","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00165a8b8\u003e: \n resource group \"np-autoscaling-d78mqd29jkc5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-d78mqd29jkc5\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fd7cab-045b-483c-9285-d64f8be28bf8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fd7cab-045b-483c-9285-d64f8be28bf8\",\n \"name\": \"17fd7cab-045b-483c-9285-d64f8be28bf8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:37:39.043957536Z\",\n \"endTime\": \"2026-09-28T10:01:44.896160867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0mtuah2idr1sqhvfksccie7qk7bk still exists (deletion dispatched at 2026-09-28T09:37:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00165a8b8\u003e: \n resource group \"np-autoscaling-d78mqd29jkc5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-d78mqd29jkc5\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fd7cab-045b-483c-9285-d64f8be28bf8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/17fd7cab-045b-483c-9285-d64f8be28bf8\",\n \"name\": \"17fd7cab-045b-483c-9285-d64f8be28bf8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:37:39.043957536Z\",\n \"endTime\": \"2026-09-28T10:01:44.896160867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0mtuah2idr1sqhvfksccie7qk7bk still exists (deletion dispatched at 2026-09-28T09:37:41Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4c7c6d61f3713ec45b4d61353a2337b492e8d3895ab05a497677e8d551e48cc5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"80098528f82c953ebcec21eb40009d3ad625bc737f3cdf6d0e0316ea06ac992c","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000fa2390\u003e: \n resource group \"rg-np-labels-taints-xb88nprh6v8r\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-xb88nprh6v8r\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a0c6ea6-d19d-46df-ab9e-9088ec00bcf2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a0c6ea6-d19d-46df-ab9e-9088ec00bcf2\",\n \"name\": \"9a0c6ea6-d19d-46df-ab9e-9088ec00bcf2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:35:33.916123706Z\",\n \"endTime\": \"2026-09-28T09:59:35.59252554Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0lp2q6dsf21b8209r6h4nf0vedlp still exists (deletion dispatched at 2026-09-28T09:35:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000fa2390\u003e: \n resource group \"rg-np-labels-taints-xb88nprh6v8r\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-xb88nprh6v8r\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a0c6ea6-d19d-46df-ab9e-9088ec00bcf2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/9a0c6ea6-d19d-46df-ab9e-9088ec00bcf2\",\n \"name\": \"9a0c6ea6-d19d-46df-ab9e-9088ec00bcf2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:35:33.916123706Z\",\n \"endTime\": \"2026-09-28T09:59:35.59252554Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0lp2q6dsf21b8209r6h4nf0vedlp still exists (deletion dispatched at 2026-09-28T09:35:37Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4d666a10c8bc5e576ecbd88c14dc35b0eb8edb74f77c0b803c9e5892ad44f24a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"d26e7881bdda81d57d491dda55f8a6f74790459a671e16de24eb0c2a3210b3d0","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a32078\u003e: \n resource group \"rg-cp-ystream-upgrade-4-22-4lhbjmsv9rt8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-47qvqf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4lhbjmsv9rt8\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e635505a-5ce4-4b08-a35e-5c1ae209525b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e635505a-5ce4-4b08-a35e-5c1ae209525b\",\n \"name\": \"e635505a-5ce4-4b08-a35e-5c1ae209525b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:37:59.959232804Z\",\n \"endTime\": \"2026-09-28T10:02:05.043620822Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ma6360plq8ebh66osj03lihjgik still exists (deletion dispatched at 2026-09-28T09:38:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a32078\u003e: \n resource group \"rg-cp-ystream-upgrade-4-22-4lhbjmsv9rt8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-47qvqf\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-4lhbjmsv9rt8\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e635505a-5ce4-4b08-a35e-5c1ae209525b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e635505a-5ce4-4b08-a35e-5c1ae209525b\",\n \"name\": \"e635505a-5ce4-4b08-a35e-5c1ae209525b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:37:59.959232804Z\",\n \"endTime\": \"2026-09-28T10:02:05.043620822Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ma6360plq8ebh66osj03lihjgik still exists (deletion dispatched at 2026-09-28T09:38:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"527cc7e7323327d12b73fed19e03a6ac95b1fedaa53bf6a673db1f82c9725f93","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"62133fbc2a990af4d75df5f3308f03fabd68fc9126fb8e338e8a039115454435","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00290a2b8\u003e: \n resource group \"rg-candidate-5-0-8h76gpwqntfk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-5z7vd6\" in resourcegroup=\"rg-candidate-5-0-8h76gpwqntfk\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf79dcd4-21ea-4d96-a412-53a56f9ed792\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf79dcd4-21ea-4d96-a412-53a56f9ed792\",\n \"name\": \"cf79dcd4-21ea-4d96-a412-53a56f9ed792\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:35:00.890511025Z\",\n \"endTime\": \"2026-09-28T09:59:04.96675702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0n0ao372vngnfec0pqf548ff4sjj still exists (deletion dispatched at 2026-09-28T09:35:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00290a2b8\u003e: \n resource group \"rg-candidate-5-0-8h76gpwqntfk\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-5z7vd6\" in resourcegroup=\"rg-candidate-5-0-8h76gpwqntfk\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf79dcd4-21ea-4d96-a412-53a56f9ed792\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf79dcd4-21ea-4d96-a412-53a56f9ed792\",\n \"name\": \"cf79dcd4-21ea-4d96-a412-53a56f9ed792\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:35:00.890511025Z\",\n \"endTime\": \"2026-09-28T09:59:04.96675702Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0n0ao372vngnfec0pqf548ff4sjj still exists (deletion dispatched at 2026-09-28T09:35:02Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"563088617a326558ab9f79dc8e1ec8adf6c1dae1eb5e85db063cdc4003c54909","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"edb70c942e30bb17da3821925bed8b5e1b90399f2f5e8e8d2a7f7e8ad2f00f79","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e523f0\u003e: \n resource group \"v20261001preview-d4mgs8lq4npr\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-d4mgs8lq4npr\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0792d6b3-7020-427d-84b3-8b07a05c3ad3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0792d6b3-7020-427d-84b3-8b07a05c3ad3\",\n \"name\": \"0792d6b3-7020-427d-84b3-8b07a05c3ad3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:27:03.486181728Z\",\n \"endTime\": \"2026-09-28T09:51:04.973500282Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u071ocvqmm5cqrijckm8sekgl81rd still exists (deletion dispatched at 2026-09-28T09:27:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e523f0\u003e: \n resource group \"v20261001preview-d4mgs8lq4npr\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"v20261001\" in resourcegroup=\"v20261001preview-d4mgs8lq4npr\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0792d6b3-7020-427d-84b3-8b07a05c3ad3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/0792d6b3-7020-427d-84b3-8b07a05c3ad3\",\n \"name\": \"0792d6b3-7020-427d-84b3-8b07a05c3ad3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:27:03.486181728Z\",\n \"endTime\": \"2026-09-28T09:51:04.973500282Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u071ocvqmm5cqrijckm8sekgl81rd still exists (deletion dispatched at 2026-09-28T09:27:05Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"60ff847956b880bcc1c241ad7ea14f180833bafaed7c24330b3e259d8b055845","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"51d9fd31471a1c0d304e788b52fdca01b1faf0db3e9c8c998e6e19021aff1d43","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001700288\u003e: \n resource group \"private-keyvault-vfws486bsfgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vfws486bsfgl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f4a43f8-8823-4442-9a9f-327ed4ef5fef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f4a43f8-8823-4442-9a9f-327ed4ef5fef\",\n \"name\": \"5f4a43f8-8823-4442-9a9f-327ed4ef5fef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:29:57.435847003Z\",\n \"endTime\": \"2026-09-28T09:54:04.986151784Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u18j4sj43hhdfhhr4o61io72f8235 still exists (deletion dispatched at 2026-09-28T09:29:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001700288\u003e: \n resource group \"private-keyvault-vfws486bsfgl\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-vfws486bsfgl\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f4a43f8-8823-4442-9a9f-327ed4ef5fef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5f4a43f8-8823-4442-9a9f-327ed4ef5fef\",\n \"name\": \"5f4a43f8-8823-4442-9a9f-327ed4ef5fef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:29:57.435847003Z\",\n \"endTime\": \"2026-09-28T09:54:04.986151784Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u18j4sj43hhdfhhr4o61io72f8235 still exists (deletion dispatched at 2026-09-28T09:29:59Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"645076889c2ae80071def05fd703806b28cd5cc077105e21dba5f463637c7dc0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"3575db45186608ce7e97cda0fd0cf74fa5dcc53098682cede51e13901add9b7f","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00186c0c0\u003e: \n resource group \"rg-np-dg-minor-4jkg9h-xz9w7xmrt69c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-4jkg9h\" in resourcegroup=\"rg-np-dg-minor-4jkg9h-xz9w7xmrt69c\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2a3184ba-a34e-4171-90e1-59c285842841\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2a3184ba-a34e-4171-90e1-59c285842841\",\n \"name\": \"2a3184ba-a34e-4171-90e1-59c285842841\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T10:04:36.362367042Z\",\n \"endTime\": \"2026-09-28T10:28:40.099696412Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3ubaokiuhrcoo558a2ghta34iks995 still exists (deletion dispatched at 2026-09-28T10:04:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00186c0c0\u003e: \n resource group \"rg-np-dg-minor-4jkg9h-xz9w7xmrt69c\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-dg-minor-4jkg9h\" in resourcegroup=\"rg-np-dg-minor-4jkg9h-xz9w7xmrt69c\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2a3184ba-a34e-4171-90e1-59c285842841\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/2a3184ba-a34e-4171-90e1-59c285842841\",\n \"name\": \"2a3184ba-a34e-4171-90e1-59c285842841\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T10:04:36.362367042Z\",\n \"endTime\": \"2026-09-28T10:28:40.099696412Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3ubaokiuhrcoo558a2ghta34iks995 still exists (deletion dispatched at 2026-09-28T10:04:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6704be487b0151427bf3cbd43056a14968f222cbdf2fb50c0f5b197a1ed4e46f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"fe6f233cf035c94775c98365cc42c5ca8d0b415b7ad76008f15194614bdb8ee8","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014aa0c0\u003e: \n resource group \"rg-cp-ystream-upgrade-4-21-tbjnlwkt5shm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-89fffp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-tbjnlwkt5shm\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/661a4e43-2a8c-46d1-b23e-1fe0392ec81c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/661a4e43-2a8c-46d1-b23e-1fe0392ec81c\",\n \"name\": \"661a4e43-2a8c-46d1-b23e-1fe0392ec81c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:31:16.445346573Z\",\n \"endTime\": \"2026-09-28T09:55:24.93685083Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u09q1a1rcl3l0m7c7el9kbge10oe7 still exists (deletion dispatched at 2026-09-28T09:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014aa0c0\u003e: \n resource group \"rg-cp-ystream-upgrade-4-21-tbjnlwkt5shm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-21-89fffp\" in resourcegroup=\"rg-cp-ystream-upgrade-4-21-tbjnlwkt5shm\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/661a4e43-2a8c-46d1-b23e-1fe0392ec81c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/661a4e43-2a8c-46d1-b23e-1fe0392ec81c\",\n \"name\": \"661a4e43-2a8c-46d1-b23e-1fe0392ec81c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:31:16.445346573Z\",\n \"endTime\": \"2026-09-28T09:55:24.93685083Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u09q1a1rcl3l0m7c7el9kbge10oe7 still exists (deletion dispatched at 2026-09-28T09:31:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"72607ac9d9053aef63cf2c1dfac9a4e836b4a8075af18ac93a134a4117c6dda3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"alert","signature_id":"2723bca6220abf5c229937295d877dffcf70869f11654d5b10ba3a9c7d1e9f35","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 16 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-28T10:33:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"cloud-network-config-controller-58dfb59986-bsjtm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2b4fc680-6a59-448e-b315-f78694553d9a\"\n Description: Pod ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/cloud-network-config-controller-58dfb59986-bsjtm (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-28T10:33:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e\", pod=\"cloud-network-config-controller-69985fdf59-fmd5d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"387b4461-aa28-4131-85f9-f2b77c4e2e0e\"\n Description: Pod ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e/cloud-network-config-controller-69985fdf59-fmd5d (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-28T10:33:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e\", pod=\"cloud-network-config-controller-69985fdf59-fmd5d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"387b4461-aa28-4131-85f9-f2b77c4e2e0e\"\n Description: Pod ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e/cloud-network-config-controller-69985fdf59-fmd5d (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-28T10:33:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e\", pod=\"csi-snapshot-controller-d4d6cd4f4-d2tqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c3f26a0c-8e66-4f75-bb2e-f2645071d798\"\n Description: Pod ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e/csi-snapshot-controller-d4d6cd4f4-d2tqr (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-28T10:33:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e\", pod=\"csi-snapshot-controller-d4d6cd4f4-d2tqr\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"c3f26a0c-8e66-4f75-bb2e-f2645071d798\"\n Description: Pod ocm-arohcpci01-2t3u0n0ao372vngnfec0pqf548ff4sjj-d9g9f6v4c3w8y9e/csi-snapshot-controller-d4d6cd4f4-d2tqr (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"csi-snapshot-controller-c9cf48cfd-vfz96\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e625ddc5-e22f-4d5c-bb2f-d4cb602de924\"\n Description: Pod ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/csi-snapshot-controller-c9cf48cfd-vfz96 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"cloud-network-config-controller-5c87577d75-pzm4d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f7312b5e-d9af-4928-8d9a-ea34f1380f1e\"\n Description: Pod ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/cloud-network-config-controller-5c87577d75-pzm4d (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"cloud-network-config-controller-5b9c45d5bc-245tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1e7400d-4c21-4a6d-a049-fcf41ebbe075\"\n Description: Pod ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/cloud-network-config-controller-5b9c45d5bc-245tg (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"cloud-network-config-controller-58dfb59986-bsjtm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"2b4fc680-6a59-448e-b315-f78694553d9a\"\n Description: Pod ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/cloud-network-config-controller-58dfb59986-bsjtm (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"csi-snapshot-controller-69d89876cf-6p78f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1a54f46c-0195-4d5e-aae8-68c97f711f84\"\n Description: Pod ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/csi-snapshot-controller-69d89876cf-6p78f (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"cloud-network-config-controller-5c87577d75-pzm4d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"f7312b5e-d9af-4928-8d9a-ea34f1380f1e\"\n Description: Pod ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/cloud-network-config-controller-5c87577d75-pzm4d (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.91:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y\", pod=\"csi-snapshot-controller-69d89876cf-6p78f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"1a54f46c-0195-4d5e-aae8-68c97f711f84\"\n Description: Pod ocm-arohcpci01-2t3u0lp2q6dsf21b8209r6h4nf0vedlp-e1u8w7m0a8d6a5y/csi-snapshot-controller-69d89876cf-6p78f (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"csi-snapshot-controller-7d6c8c6fbb-xmtc5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"07ef49a6-b3c7-427b-ab6f-44204ea64710\"\n Description: Pod ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/csi-snapshot-controller-7d6c8c6fbb-xmtc5 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b\", pod=\"csi-snapshot-controller-c9cf48cfd-vfz96\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e625ddc5-e22f-4d5c-bb2f-d4cb602de924\"\n Description: Pod ocm-arohcpci01-2t3u0ffhfsef977jtdd70rv0ev42cvrg-k2l3d4g2t9f1a4b/csi-snapshot-controller-c9cf48cfd-vfz96 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"cloud-network-config-controller-5b9c45d5bc-245tg\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e1e7400d-4c21-4a6d-a049-fcf41ebbe075\"\n Description: Pod ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/cloud-network-config-controller-5b9c45d5bc-245tg (controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-09-28T10:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1041152-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.118:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1041152/providers/microsoft.monitor/accounts/hcps-j1041152\", namespace=\"ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n\", pod=\"csi-snapshot-controller-7d6c8c6fbb-xmtc5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"07ef49a6-b3c7-427b-ab6f-44204ea64710\"\n Description: Pod ocm-arohcpci01-2t3u0kbt737hlqiqbf1dq3vg54p17efo-n1w7s8z8q6s5h2n/csi-snapshot-controller-7d6c8c6fbb-xmtc5 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"76609624f7385be53e6b70c8e479defc421efeedff13c52b52726cdff012a39e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"d992f3cd9ef02517f10c183a1d637491540983decdd93733599aca19cf494b86","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:299]: failed to update cluster with new KMS key with the second rotation\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00123cdb0\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sfghgk8d4rdx\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-sfghgk8d4rdx, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:299]: failed to update cluster with new KMS key with the second rotation\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc00123cdb0\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sfghgk8d4rdx\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-sfghgk8d4rdx, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sfghgk8d4rdx\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-sfghgk8d4rdx, error: context deadline exceeded"}},{"row_id":"8685e54589883527e7c0d838488991de33f4b806ba215f8fa6e945156fae74ea","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"f86aa345da5041b62d0094051db665216f09f00381a536f35956b7dd0f86e650","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002212048\u003e: \n resource group \"e2e-cidr-connectivity-qw9smztvq7ws\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qw9smztvq7ws\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/304f8159-6c5e-4d11-8b14-b44d5bcb0d51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/304f8159-6c5e-4d11-8b14-b44d5bcb0d51\",\n \"name\": \"304f8159-6c5e-4d11-8b14-b44d5bcb0d51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:38:13.479803853Z\",\n \"endTime\": \"2026-09-28T10:02:14.975908515Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u121rl1nqjoihtgis9hmujce91fc8 still exists (deletion dispatched at 2026-09-28T09:38:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002212048\u003e: \n resource group \"e2e-cidr-connectivity-qw9smztvq7ws\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-qw9smztvq7ws\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/304f8159-6c5e-4d11-8b14-b44d5bcb0d51\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/304f8159-6c5e-4d11-8b14-b44d5bcb0d51\",\n \"name\": \"304f8159-6c5e-4d11-8b14-b44d5bcb0d51\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:38:13.479803853Z\",\n \"endTime\": \"2026-09-28T10:02:14.975908515Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u121rl1nqjoihtgis9hmujce91fc8 still exists (deletion dispatched at 2026-09-28T09:38:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"896145bb00f1e8461584bebc323bac400033fd70f5c026763b877d5c86564cf6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"4d469d9ff8433380f5c21911bcde986dbc7542a7445ebb8ef34554faed2520d6","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002e96258\u003e: \n resource group \"nodepool-update-nodes-pwhzr9kc5fmg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-pwhzr9kc5fmg\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c679bb39-251f-482e-aac7-5974edb2fbba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c679bb39-251f-482e-aac7-5974edb2fbba\",\n \"name\": \"c679bb39-251f-482e-aac7-5974edb2fbba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:48:09.967265057Z\",\n \"endTime\": \"2026-09-28T10:12:09.567399351Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u1fimurv5ioj3idg4r827ncmq2om6 still exists (deletion dispatched at 2026-09-28T09:48:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002e96258\u003e: \n resource group \"nodepool-update-nodes-pwhzr9kc5fmg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-pwhzr9kc5fmg\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c679bb39-251f-482e-aac7-5974edb2fbba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c679bb39-251f-482e-aac7-5974edb2fbba\",\n \"name\": \"c679bb39-251f-482e-aac7-5974edb2fbba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:48:09.967265057Z\",\n \"endTime\": \"2026-09-28T10:12:09.567399351Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u1fimurv5ioj3idg4r827ncmq2om6 still exists (deletion dispatched at 2026-09-28T09:48:11Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters, 2 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 2 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a0d57f02af9316827a2a2fc221b14182116670ecb28ee84f13db8abb179b306","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"b50a899b542ef0c196a7d92efe90960659f5d0b7a62b38d8e35564b620d606b8","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000b82090\u003e: \n resource group \"rg-zstream-upgrade-5-0-g9ntqxcmp6cx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-msh4hf\" in resourcegroup=\"rg-zstream-upgrade-5-0-g9ntqxcmp6cx\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5007d355-1390-447c-a836-e7201fdc5f8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5007d355-1390-447c-a836-e7201fdc5f8f\",\n \"name\": \"5007d355-1390-447c-a836-e7201fdc5f8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:21.326906807Z\",\n \"endTime\": \"2026-09-28T10:00:25.344782427Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0kbt737hlqiqbf1dq3vg54p17efo still exists (deletion dispatched at 2026-09-28T09:36:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000b82090\u003e: \n resource group \"rg-zstream-upgrade-5-0-g9ntqxcmp6cx\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-5-0-msh4hf\" in resourcegroup=\"rg-zstream-upgrade-5-0-g9ntqxcmp6cx\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5007d355-1390-447c-a836-e7201fdc5f8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5007d355-1390-447c-a836-e7201fdc5f8f\",\n \"name\": \"5007d355-1390-447c-a836-e7201fdc5f8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:21.326906807Z\",\n \"endTime\": \"2026-09-28T10:00:25.344782427Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0kbt737hlqiqbf1dq3vg54p17efo still exists (deletion dispatched at 2026-09-28T09:36:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ec8c784fb0e671167a1df035f1a556a9ad48d3e490a6fc8ebd1e074021793ba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"1dabd48da26bb8f586d188358c2a6b848886de5984190e3c86cb8de205e943bb","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000eb40f0\u003e: \n resource group \"fips-enabled-bn5q2wwsf7rp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn5q2wwsf7rp\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\",\n \"name\": \"3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:05.789845252Z\",\n \"endTime\": \"2026-09-28T09:54:14.850550256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0hvkv6dq5otq5pkoot1lod893ptj still exists (deletion dispatched at 2026-09-28T09:30:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000eb40f0\u003e: \n resource group \"fips-enabled-bn5q2wwsf7rp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bn5q2wwsf7rp\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\",\n \"name\": \"3a6f097a-b7cb-40aa-8dcb-c39bec4a86df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:05.789845252Z\",\n \"endTime\": \"2026-09-28T09:54:14.850550256Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0hvkv6dq5otq5pkoot1lod893ptj still exists (deletion dispatched at 2026-09-28T09:30:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"957cf19933603ba736ec4710a37c866ab5abffa6f918f4b6e8c5f98806f03353","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"9750f33ed5e6147a22d180f7699d450888db4a731961f85f26cda84a3169792e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0021100f0\u003e: \n resource group \"rg-candidate-4-22-rsthdhx2w4z7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hqvm4n\" in resourcegroup=\"rg-candidate-4-22-rsthdhx2w4z7\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf083a7d-366e-461c-b95d-686d145e886a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf083a7d-366e-461c-b95d-686d145e886a\",\n \"name\": \"cf083a7d-366e-461c-b95d-686d145e886a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:32:16.310177849Z\",\n \"endTime\": \"2026-09-28T09:56:25.119461348Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0o5ou7tq797vcmg5v755s0q2tngv still exists (deletion dispatched at 2026-09-28T09:32:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0021100f0\u003e: \n resource group \"rg-candidate-4-22-rsthdhx2w4z7\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-hqvm4n\" in resourcegroup=\"rg-candidate-4-22-rsthdhx2w4z7\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf083a7d-366e-461c-b95d-686d145e886a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cf083a7d-366e-461c-b95d-686d145e886a\",\n \"name\": \"cf083a7d-366e-461c-b95d-686d145e886a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:32:16.310177849Z\",\n \"endTime\": \"2026-09-28T09:56:25.119461348Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0o5ou7tq797vcmg5v755s0q2tngv still exists (deletion dispatched at 2026-09-28T09:32:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"96072e277a4c695841c43f544988993e1abf9b3291018f6bd05f9555d7230347","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"alert","signature_id":"4b99f55b3dddceb435e0a52e1d7443cc1d1476dd1512466b15df642e807fd310","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T09:36:38Z\n Ended: 2026-09-28T09:42:38Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j1041152-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3u0ioqrfjsn6acu7pjpg12lkcddspk-i4m7g3h8t9f6r1s\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j1041152-mgmt-2\nNamespace: ocm-arohcpci01-2t3u0ioqrfjsn6acu7pjpg12lkcddspk-i4m7g3h8t9f6r1s\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T10:21:38Z\n Ended: 2026-09-28T10:28:38Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j1041152-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3u0ma6360plq8ebh66osj03lihjgik-j3g2n6z5l7j6d3s\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j1041152-mgmt-1\nNamespace: ocm-arohcpci01-2t3u0ma6360plq8ebh66osj03lihjgik-j3g2n6z5l7j6d3s","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"9e4f0eb491b1613df3ebb48459c7cecf547dcd3e6a37661a0a6eaba50221c719","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"a69c8acf0abd80d165248bcaf55a4ced12cad424d219715d2c11e129770e5945","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000fca138\u003e: \n resource group \"rg-cordon-test-29mmt5q6w9zq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-29mmt5q6w9zq\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d26f882f-0351-488a-bc80-3a658e3574f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d26f882f-0351-488a-bc80-3a658e3574f5\",\n \"name\": \"d26f882f-0351-488a-bc80-3a658e3574f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:21.548016661Z\",\n \"endTime\": \"2026-09-28T09:54:24.966223651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ojhe2srssobiigo2u2jk64kd72f still exists (deletion dispatched at 2026-09-28T09:30:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000fca138\u003e: \n resource group \"rg-cordon-test-29mmt5q6w9zq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cordon-hcp-cluster\" in resourcegroup=\"rg-cordon-test-29mmt5q6w9zq\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d26f882f-0351-488a-bc80-3a658e3574f5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d26f882f-0351-488a-bc80-3a658e3574f5\",\n \"name\": \"d26f882f-0351-488a-bc80-3a658e3574f5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:21.548016661Z\",\n \"endTime\": \"2026-09-28T09:54:24.966223651Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ojhe2srssobiigo2u2jk64kd72f still exists (deletion dispatched at 2026-09-28T09:30:26Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9f4e73420467be65896174eaeab58ee781e17f692f5321d56a5b5d899aca1109","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"e35a27569175c5bae78b2e4e738387abcfc6661dc3d77070c5b5ec75ed06600a","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00180c2e8\u003e: \n resource group \"rg-negative-tests-rsx9k9dxrxnr\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-rsx9k9dxrxnr\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1446725-d527-4a08-af82-4288738118c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1446725-d527-4a08-af82-4288738118c3\",\n \"name\": \"b1446725-d527-4a08-af82-4288738118c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:43:15.547591968Z\",\n \"endTime\": \"2026-09-28T10:07:19.754648726Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3ubb559kv9knv31rsut3pvknqg6pg8 still exists (deletion dispatched at 2026-09-28T09:43:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00180c2e8\u003e: \n resource group \"rg-negative-tests-rsx9k9dxrxnr\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-rsx9k9dxrxnr\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1446725-d527-4a08-af82-4288738118c3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1446725-d527-4a08-af82-4288738118c3\",\n \"name\": \"b1446725-d527-4a08-af82-4288738118c3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:43:15.547591968Z\",\n \"endTime\": \"2026-09-28T10:07:19.754648726Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3ubb559kv9knv31rsut3pvknqg6pg8 still exists (deletion dispatched at 2026-09-28T09:43:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a6f9d3290c6203f4c79fad405d365489dfab77e64137f807f91427219759158a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"41f8565c8d66121e0492feecad4bd386abcae6dbe95e544be1df719574ccf61b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002958120\u003e: \n resource group \"rg-candidate-4-20-n6x2rfcpmr5t\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-bkc8nt\" in resourcegroup=\"rg-candidate-4-20-n6x2rfcpmr5t\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d443cce-28d6-4421-9f77-bc36ad9031f7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d443cce-28d6-4421-9f77-bc36ad9031f7\",\n \"name\": \"4d443cce-28d6-4421-9f77-bc36ad9031f7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:44.717303002Z\",\n \"endTime\": \"2026-09-28T09:54:45.21106923Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ce2mv9aje3edhhstj5ffjhl0mpi still exists (deletion dispatched at 2026-09-28T09:31:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc002958120\u003e: \n resource group \"rg-candidate-4-20-n6x2rfcpmr5t\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-bkc8nt\" in resourcegroup=\"rg-candidate-4-20-n6x2rfcpmr5t\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d443cce-28d6-4421-9f77-bc36ad9031f7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/4d443cce-28d6-4421-9f77-bc36ad9031f7\",\n \"name\": \"4d443cce-28d6-4421-9f77-bc36ad9031f7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:44.717303002Z\",\n \"endTime\": \"2026-09-28T09:54:45.21106923Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0ce2mv9aje3edhhstj5ffjhl0mpi still exists (deletion dispatched at 2026-09-28T09:31:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b6497feab000161cefb91c4f1b313ea921a04bb8c2169e7bb08ce99a423eb1cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"2dfb5050c6aeff4f9fb910a04a4f8308c1afbe380de422b5d8c1203d2407f57e","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00138e0d8\u003e: \n resource group \"rg-np-downgrade-z882qs-gvknqpcq28h4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-z882qs\" in resourcegroup=\"rg-np-downgrade-z882qs-gvknqpcq28h4\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caadaa7a-696b-4362-b133-bd079c0d1241\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caadaa7a-696b-4362-b133-bd079c0d1241\",\n \"name\": \"caadaa7a-696b-4362-b133-bd079c0d1241\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:47:47.000063819Z\",\n \"endTime\": \"2026-09-28T10:11:49.934874142Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3ubdmtg6c4vit45bipbt111i5ntkei still exists (deletion dispatched at 2026-09-28T09:47:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00138e0d8\u003e: \n resource group \"rg-np-downgrade-z882qs-gvknqpcq28h4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-z882qs\" in resourcegroup=\"rg-np-downgrade-z882qs-gvknqpcq28h4\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caadaa7a-696b-4362-b133-bd079c0d1241\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/caadaa7a-696b-4362-b133-bd079c0d1241\",\n \"name\": \"caadaa7a-696b-4362-b133-bd079c0d1241\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:47:47.000063819Z\",\n \"endTime\": \"2026-09-28T10:11:49.934874142Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3ubdmtg6c4vit45bipbt111i5ntkei still exists (deletion dispatched at 2026-09-28T09:47:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b68705f41ae387aa7806187b86690c18da65eb70b21a44d715600308fa41c7ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"98ef78430baca42d6f526e7ecca494ec92e181a87375c4f938b1a9850f7a6bab","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011a84b0\u003e: \n resource group \"rg-zstream-upgrade-4-22-wc2zsjqkql5z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-pwm7lf\" in resourcegroup=\"rg-zstream-upgrade-4-22-wc2zsjqkql5z\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b020a635-70f4-4fa3-bbeb-15a20c919dd5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b020a635-70f4-4fa3-bbeb-15a20c919dd5\",\n \"name\": \"b020a635-70f4-4fa3-bbeb-15a20c919dd5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:33:46.781992368Z\",\n \"endTime\": \"2026-09-28T09:57:54.928017678Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0gnt93pnblunkan391ssod5p70ip still exists (deletion dispatched at 2026-09-28T09:34:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011a84b0\u003e: \n resource group \"rg-zstream-upgrade-4-22-wc2zsjqkql5z\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-22-pwm7lf\" in resourcegroup=\"rg-zstream-upgrade-4-22-wc2zsjqkql5z\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b020a635-70f4-4fa3-bbeb-15a20c919dd5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b020a635-70f4-4fa3-bbeb-15a20c919dd5\",\n \"name\": \"b020a635-70f4-4fa3-bbeb-15a20c919dd5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:33:46.781992368Z\",\n \"endTime\": \"2026-09-28T09:57:54.928017678Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0gnt93pnblunkan391ssod5p70ip still exists (deletion dispatched at 2026-09-28T09:34:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bdce4cd71a5a658af92b81a05a1f04e56498c424781be1e7f1b89969378305a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"d0ecd7d267b8b497361e55a6ad824a7f4015da5efc239b874ba4bc345a393ad9","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010b4108\u003e: \n resource group \"rg-zstream-upgrade-4-21-n75m8pfcjsx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-kjvjjk\" in resourcegroup=\"rg-zstream-upgrade-4-21-n75m8pfcjsx5\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45231285-274d-43cd-98fa-1c2da5982b8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45231285-274d-43cd-98fa-1c2da5982b8d\",\n \"name\": \"45231285-274d-43cd-98fa-1c2da5982b8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:34.341653398Z\",\n \"endTime\": \"2026-09-28T10:00:35.039725799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0d23l1qdtpftb5407dlteafc6gfr still exists (deletion dispatched at 2026-09-28T09:36:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0010b4108\u003e: \n resource group \"rg-zstream-upgrade-4-21-n75m8pfcjsx5\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-kjvjjk\" in resourcegroup=\"rg-zstream-upgrade-4-21-n75m8pfcjsx5\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45231285-274d-43cd-98fa-1c2da5982b8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/45231285-274d-43cd-98fa-1c2da5982b8d\",\n \"name\": \"45231285-274d-43cd-98fa-1c2da5982b8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:34.341653398Z\",\n \"endTime\": \"2026-09-28T10:00:35.039725799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0d23l1qdtpftb5407dlteafc6gfr still exists (deletion dispatched at 2026-09-28T09:36:35Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d0aaf73b1ed774e2a0a9f8c3f4789012afa042a995a5e80e169c4ec1d829c98f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"alert","signature_id":"a7a4c6a0026c3f599f6b4f74fe252b7b2b73f3f52e4f6fcefc52ad3c0977d311","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:24:44Z\n Ended: 2026-09-28T10:37:42Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j1041152-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3ubdmtg6c4vit45bipbt111i5ntkei-m5v3h7g8t9d4c9v\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j1041152-mgmt-1\nNamespace: ocm-arohcpci01-2t3ubdmtg6c4vit45bipbt111i5ntkei-m5v3h7g8t9d4c9v","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"d21a5061f79949f43640e3a213eeaf83724dd3c81cf8590deb967e63db56e228","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"alert","signature_id":"60ce8c4e66e5aca104e5530423691fb540d7e157b22d3d53163288d5c65dca6b","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-28T10:24:42Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j1041152-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3u0ma6360plq8ebh66osj03lihjgik-j3g2n6z5l7j6d3s\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j1041152-mgmt-1\nNamespace: ocm-arohcpci01-2t3u0ma6360plq8ebh66osj03lihjgik-j3g2n6z5l7j6d3s","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"d7428ab2b8c608fc4344fdba5234dc9d4178154db9d6cfab67452067cfcded17","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"d4bf4d27b2b15cf42ec27f335cd2d5cb9602474ad668a2b068ea83dd235f1bb1","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001438840\u003e: \n resource group \"rg-np-version-upgrade-gp7ddw-8fkxplz5m659\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-gp7ddw\" in resourcegroup=\"rg-np-version-upgrade-gp7ddw-8fkxplz5m659\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91fd8806-f596-45a2-8056-737eddeb0c8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91fd8806-f596-45a2-8056-737eddeb0c8d\",\n \"name\": \"91fd8806-f596-45a2-8056-737eddeb0c8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:43:50.208665116Z\",\n \"endTime\": \"2026-09-28T10:07:59.865594069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u1fdl8l8ls86c304u4oqijmkekk18 still exists (deletion dispatched at 2026-09-28T09:43:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001438840\u003e: \n resource group \"rg-np-version-upgrade-gp7ddw-8fkxplz5m659\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-gp7ddw\" in resourcegroup=\"rg-np-version-upgrade-gp7ddw-8fkxplz5m659\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91fd8806-f596-45a2-8056-737eddeb0c8d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/91fd8806-f596-45a2-8056-737eddeb0c8d\",\n \"name\": \"91fd8806-f596-45a2-8056-737eddeb0c8d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:43:50.208665116Z\",\n \"endTime\": \"2026-09-28T10:07:59.865594069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u1fdl8l8ls86c304u4oqijmkekk18 still exists (deletion dispatched at 2026-09-28T09:43:51Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e40cd552ae34b06074bf1f830f61ef714fe3bbaf0b732ae3660455ffaecc0f8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"febfa4ad19c12a01fa699d939f8ef5e4e624851c9e2c5e88835d04e4a4a57859","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a7e048\u003e: \n resource group \"rg-cp-ystream-upgrade-5-0-q52nfkvp5d5w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-895w8v\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-q52nfkvp5d5w\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/764e0c2e-ef4e-439a-97b4-27c1b0eec183\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/764e0c2e-ef4e-439a-97b4-27c1b0eec183\",\n \"name\": \"764e0c2e-ef4e-439a-97b4-27c1b0eec183\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:39.848641766Z\",\n \"endTime\": \"2026-09-28T10:00:44.979089844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0jpa4etua5lae503qhk8taguuqgn still exists (deletion dispatched at 2026-09-28T09:36:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a7e048\u003e: \n resource group \"rg-cp-ystream-upgrade-5-0-q52nfkvp5d5w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-5-0-895w8v\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-q52nfkvp5d5w\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/764e0c2e-ef4e-439a-97b4-27c1b0eec183\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/764e0c2e-ef4e-439a-97b4-27c1b0eec183\",\n \"name\": \"764e0c2e-ef4e-439a-97b4-27c1b0eec183\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:36:39.848641766Z\",\n \"endTime\": \"2026-09-28T10:00:44.979089844Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0jpa4etua5lae503qhk8taguuqgn still exists (deletion dispatched at 2026-09-28T09:36:42Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ecb16735dd7bba0ad93ada637c6e5f8b70cf75f1b5b35f5c13ebcd6483a20573","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"48212f8bd40f9c5ee7d2e207dd0fc8d883c102f5d64b6cb526bcca292b3142d8","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a02240\u003e: \n resource group \"rg-np-version-upgrade-scljgc-q96m77tgzfb6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-scljgc\" in resourcegroup=\"rg-np-version-upgrade-scljgc-q96m77tgzfb6\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1103c4f-e224-4b48-b9f0-0980b852bf71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1103c4f-e224-4b48-b9f0-0980b852bf71\",\n \"name\": \"d1103c4f-e224-4b48-b9f0-0980b852bf71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:42:29.944025Z\",\n \"endTime\": \"2026-09-28T10:06:29.757432329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0onupvo7hktmjvhi336mbs8af0cq still exists (deletion dispatched at 2026-09-28T09:43:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a02240\u003e: \n resource group \"rg-np-version-upgrade-scljgc-q96m77tgzfb6\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-version-upgrade-cluster-scljgc\" in resourcegroup=\"rg-np-version-upgrade-scljgc-q96m77tgzfb6\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1103c4f-e224-4b48-b9f0-0980b852bf71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/d1103c4f-e224-4b48-b9f0-0980b852bf71\",\n \"name\": \"d1103c4f-e224-4b48-b9f0-0980b852bf71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:42:29.944025Z\",\n \"endTime\": \"2026-09-28T10:06:29.757432329Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0onupvo7hktmjvhi336mbs8af0cq still exists (deletion dispatched at 2026-09-28T09:43:01Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f7ca8a35b8a4a617b760ad042155870e889bdb9af617e2edd57162f9c506a287","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"alert","signature_id":"6789e672fb7d7a5c3e1c55f770176fb00fe56be9b227fabded289671c9ee231b","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T10:21:38Z\n Ended: 2026-09-28T10:28:43Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j1041152-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3ubdmtg6c4vit45bipbt111i5ntkei-m5v3h7g8t9d4c9v\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j1041152-mgmt-1\nNamespace: ocm-arohcpci01-2t3ubdmtg6c4vit45bipbt111i5ntkei-m5v3h7g8t9d4c9v","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"fa1e135efa669e4a395c61cc43904708ee9908c34a977e9c8b5f7c288464332a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6900/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104485944341041152","occurred_at":"2026-09-28T08:18:43Z","failed_at":"e2e","signature_id":"5dfe14507d8b11e66e113439e7fd5e5658e11aa6bef04974f9ae810928fd2cab","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015461e0\u003e: \n resource group \"rg-candidate-4-23-wmbvtjdttxwg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-jt55tk\" in resourcegroup=\"rg-candidate-4-23-wmbvtjdttxwg\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae8af0bb-8a34-470d-bea3-06838637e747\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae8af0bb-8a34-470d-bea3-06838637e747\",\n \"name\": \"ae8af0bb-8a34-470d-bea3-06838637e747\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:46.693994314Z\",\n \"endTime\": \"2026-09-28T09:54:55.540074205Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0k121hdg8khh203rm14pi2dgi08b still exists (deletion dispatched at 2026-09-28T09:30:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015461e0\u003e: \n resource group \"rg-candidate-4-23-wmbvtjdttxwg\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-jt55tk\" in resourcegroup=\"rg-candidate-4-23-wmbvtjdttxwg\" to finish deleting: GET https://rp.j1041152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae8af0bb-8a34-470d-bea3-06838637e747\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ae8af0bb-8a34-470d-bea3-06838637e747\",\n \"name\": \"ae8af0bb-8a34-470d-bea3-06838637e747\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T09:30:46.693994314Z\",\n \"endTime\": \"2026-09-28T09:54:55.540074205Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3u0k121hdg8khh203rm14pi2dgi08b still exists (deletion dispatched at 2026-09-28T09:30:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":41,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","fafb6c3b0b1222d20f3a2dd1d4373bedcec1a356cddca02088bb7d86a61689f1"],"matched_occurrences":41,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104484510623076352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7071,"pr_state":"open","pr_sha":"927e534e1a98019d0a82c229df179a5455cd788c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T08:13:01Z","started_at":"2026-09-28T08:13:01Z","completed_at":"2026-09-28T10:18:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T10:29:26.14966072Z","timing_metadata_checked_at":"2026-09-28T10:29:26.14966072Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T10:29:26.152184412Z","region_metadata_checked_at":"2026-09-28T10:29:26.152184412Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"0e70a3b420ca263ca40dfb8f509e249621c2273b8204edaada6fa84ad206d9ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104484510623076352","occurred_at":"2026-09-28T08:13:01Z","failed_at":"alert","signature_id":"bb576c3e3075ea38f79306c173cad7ebcec01d4df44d1dd35aed1e3a8f78bd19","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T09:00:41Z\n Ended: 2026-09-28T09:37:40Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j3076352-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3totsphp1krc605sa3296gqttbfa03-l4x6j4k8k1d0k5l\", region=\"centralus\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j3076352-mgmt-1\nNamespace: ocm-arohcpci01-2t3totsphp1krc605sa3296gqttbfa03-l4x6j4k8k1d0k5l\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T09:02:40Z\n Ended: 2026-09-28T09:11:43Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j3076352-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3toqtlnbhe7378oukj6o7c79jqcnoj-b6u2o5n7c3y1y3q\", region=\"centralus\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j3076352-mgmt-2\nNamespace: ocm-arohcpci01-2t3toqtlnbhe7378oukj6o7c79jqcnoj-b6u2o5n7c3y1y3q","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"fb34c704fda1da3b120caa696c59a75f0c66f51f374cd81e48826745d199d4b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7071/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104484510623076352","occurred_at":"2026-09-28T08:13:01Z","failed_at":"alert","signature_id":"2d8ac31341ad08baa5d71a8dcae5ad28f7794f7d50db6c4ebde370cafd3d436c","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T09:03:45Z\n Ended: 2026-09-28T10:02:42Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j3076352-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3totsphp1krc605sa3296gqttbfa03-l4x6j4k8k1d0k5l\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j3076352-mgmt-1\nNamespace: ocm-arohcpci01-2t3totsphp1krc605sa3296gqttbfa03-l4x6j4k8k1d0k5l\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T09:05:46Z\n Ended: 2026-09-28T09:35:42Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j3076352-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3toqtlnbhe7378oukj6o7c79jqcnoj-b6u2o5n7c3y1y3q\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j3076352-mgmt-2\nNamespace: ocm-arohcpci01-2t3toqtlnbhe7378oukj6o7c79jqcnoj-b6u2o5n7c3y1y3q","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104484432940371968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6226,"pr_state":"closed","pr_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","final_merged_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T08:12:42Z","started_at":"2026-09-28T08:12:42Z","completed_at":"2026-09-28T10:19:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T10:29:25.933445904Z","timing_metadata_checked_at":"2026-09-28T10:29:25.933445904Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T10:29:25.936353895Z","region_metadata_checked_at":"2026-09-28T10:29:25.936353895Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"ca0dc527140da3083a50b162a8b913d668050823696f51600694276071a33e56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104484432940371968","occurred_at":"2026-09-28T08:12:42Z","failed_at":"alert","signature_id":"1200d6401a7d362c5bf39c4833d4b21acde0146bd26ab3743b40cda347a928f3","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T09:32:29Z\n Ended: 2026-09-28T09:38:24Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j0371968-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j0371968-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104483794588274688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7158,"pr_state":"open","pr_sha":"e123bf84bed89cda170c76f44f977013129906a0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T08:10:10Z","started_at":"2026-09-28T08:10:10Z","completed_at":"2026-09-28T10:09:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T10:19:10.211446405Z","timing_metadata_checked_at":"2026-09-28T10:19:10.211446405Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T10:19:10.214204896Z","region_metadata_checked_at":"2026-09-28T10:19:10.214204896Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"33a327fba5fe00bf078fc27f427d4e82a6c2bb8bf96430ded370342b331dfc02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7158/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104483794588274688","occurred_at":"2026-09-28T08:10:10Z","failed_at":"alert","signature_id":"6b427d606ce250ce673bfc708c7544504d7aa9cf46f968e353a45fe179cdbe65","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T09:18:31Z\n Ended: 2026-09-28T09:30:33Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8274688-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8274688-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104475990993408000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6979,"pr_state":"closed","pr_sha":"562ef1971e2f5a6c07dd80367b6d9070397b1a93","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-28T07:39:10Z","started_at":"2026-09-28T07:39:10Z","completed_at":"2026-09-28T10:01:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T10:09:00.173740323Z","timing_metadata_checked_at":"2026-09-28T10:09:00.173740323Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T10:09:00.176951313Z","region_metadata_checked_at":"2026-09-28T10:09:00.176951313Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7216/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104472130388234240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7216,"pr_state":"closed","pr_sha":"2296068e500497f975c8bd63d27c3c90c79e949f","final_merged_sha":"2296068e500497f975c8bd63d27c3c90c79e949f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T07:23:49Z","started_at":"2026-09-28T07:23:49Z","completed_at":"2026-09-28T09:37:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T09:47:55.527654052Z","timing_metadata_checked_at":"2026-09-28T09:47:55.527654052Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T09:47:55.530161744Z","region_metadata_checked_at":"2026-09-28T09:47:55.530161744Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"23064318668cfbe0f9cc07b4853d8dea194c6eb83a2ce7ce540e5f3cf3c87fcc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7216/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104472130388234240","occurred_at":"2026-09-28T07:23:49Z","failed_at":"alert","signature_id":"c01af702f03601a1f0d4848381ba9bdaa5bf0ccf7799bd12a17435bb2cf8037c","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T08:45:52Z\n Ended: 2026-09-28T08:51:55Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8234240-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8234240-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104468013976653824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7215,"pr_state":"open","pr_sha":"4c3b3d5d645acb69fa7458f18c9319f14d4142d1","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T07:07:28Z","started_at":"2026-09-28T07:07:28Z","completed_at":"2026-09-28T09:17:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T09:26:46.32245386Z","timing_metadata_checked_at":"2026-09-28T09:26:46.32245386Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T09:26:46.325581451Z","region_metadata_checked_at":"2026-09-28T09:26:46.325581451Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"6bd9870aeb9f89e052a1bfca26b25063e3e246bf0a0381259d731fe6145bb5ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104468013976653824","occurred_at":"2026-09-28T07:07:28Z","failed_at":"alert","signature_id":"b3655bb5b695bdc2e0832b9062b0fa6d3aa5dfa0ae58d7d00560b741e3fab4a2","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T08:07:37Z\n Ended: 2026-09-28T08:55:37Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j6653824-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3stml6c3gc3s82ndnpr0dtik24as4p-q2l2i3u5g4g6x0r\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j6653824-mgmt-2\nNamespace: ocm-arohcpci01-2t3stml6c3gc3s82ndnpr0dtik24as4p-q2l2i3u5g4g6x0r","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"b521da882994a3a61f5d7d52d557c8da2f31fea3817ec242601fc474e60e8af2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7215/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104468013976653824","occurred_at":"2026-09-28T07:07:28Z","failed_at":"alert","signature_id":"95dae5d8d35052e7be6830fe4168275bb671f71aab92c05a8181527ef6c63799","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T08:04:37Z\n Ended: 2026-09-28T08:30:38Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j6653824-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3stml6c3gc3s82ndnpr0dtik24as4p-q2l2i3u5g4g6x0r\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j6653824-mgmt-2\nNamespace: ocm-arohcpci01-2t3stml6c3gc3s82ndnpr0dtik24as4p-q2l2i3u5g4g6x0r","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104466453632651264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7214,"pr_state":"closed","pr_sha":"348364a54559dcede0e74a9e4e09def229be2488","final_merged_sha":"2ae778845f66d98316f4f41951553bf09a1e19db","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T07:01:11Z","started_at":"2026-09-28T07:01:11Z","completed_at":"2026-09-28T09:32:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T09:37:35.769069815Z","timing_metadata_checked_at":"2026-09-28T09:37:35.769069815Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T09:37:35.774055399Z","region_metadata_checked_at":"2026-09-28T09:37:35.774055399Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"425fb2d198e355b9000030e126eff7797c879a0e5b604dabdcaa6a0b5ed98a15","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104466453632651264","occurred_at":"2026-09-28T07:01:11Z","failed_at":"alert","signature_id":"f0862ed51a1f4c7c2323f68ca122f5ce11e850a716c3b7c2fa2bb259aced13b0","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T08:14:08Z\n Ended: 2026-09-28T08:38:09Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j2651264-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3t2a1queqj7uhi7vm5f3spuqd988nl-private-kas\", region=\"centralus\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2651264-mgmt-2\nNamespace: ocm-arohcpci01-2t3t2a1queqj7uhi7vm5f3spuqd988nl-private-kas","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"5588c6f1883fec9b0b2a9d2c2d1253ed528c1840ddbc766ed7f4c64b27dbffb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104466453632651264","occurred_at":"2026-09-28T07:01:11Z","failed_at":"alert","signature_id":"ec67058d65d185cbb1282193881a768a798758cdd0d0444d20953e4ea53d053f","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T08:12:21Z\n Ended: 2026-09-28T08:44:23Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2651264-svc\", component=\"frontend\", method=\"put\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2651264-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"e5b4c7ebe374b977e634fa6b4f7facbbeb628560ff233bfc63bcb6d393efecb1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7214/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104466453632651264","occurred_at":"2026-09-28T07:01:11Z","failed_at":"alert","signature_id":"19f23baf6d87ff64419ebe3fa5185da1fce2564267135e1650412b08bae427b9","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T08:17:15Z\n Ended: 2026-09-28T09:03:09Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j2651264-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3t2a1queqj7uhi7vm5f3spuqd988nl-private-kas\", region=\"centralus\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2651264-mgmt-2\nNamespace: ocm-arohcpci01-2t3t2a1queqj7uhi7vm5f3spuqd988nl-private-kas","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"942d17a70a130ecc74fc8c2a9727ab849836bd52","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T06:24:41Z","started_at":"2026-09-28T06:24:41Z","completed_at":"2026-09-28T08:47:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T08:55:36.202730232Z","timing_metadata_checked_at":"2026-09-28T08:55:36.202730232Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T08:55:36.206110222Z","region_metadata_checked_at":"2026-09-28T08:55:36.206110222Z"},"failed_at":["alert","e2e"],"failed_test_count":16,"occurrences":[{"row_id":"1f2665d7497abc6f9b0e6da8b97eeb8c9412abbfdd154f637bc2b59c2ab0f19a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"e2e","signature_id":"8f8369e8e139f64a46a3b6dd8d438f715d6bab665ea52bac944061720f9f66e7","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-rxbdcb\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090b40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-rxbdcb: failed waiting for cluster=\"cp-ystream-upgrade-4-23-rxbdcb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-jvcp8dmnbrt7\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e7e2861-4689-447d-b71c-e2cfaeaee7cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e7e2861-4689-447d-b71c-e2cfaeaee7cf\",\n \"name\": \"1e7e2861-4689-447d-b71c-e2cfaeaee7cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:17:56.829077997Z\",\n \"endTime\": \"2026-09-28T07:47:03.158495671Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.t0n9t0u4r4r5j6y.biea.j4259200.hcp.osadev.cloud failed: Get \\\"https://api-int.t0n9t0u4r4r5j6y.biea.j4259200.hcp.osadev.cloud:443/healthz?verbose\\\": context deadline exceeded (Client.Timeout exceeded while awaiting headers); hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-23-rxbdcb\" at install version 4.22\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001090b40\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-23-rxbdcb: failed waiting for cluster=\"cp-ystream-upgrade-4-23-rxbdcb\" in resourcegroup=\"rg-cp-ystream-upgrade-4-23-jvcp8dmnbrt7\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e7e2861-4689-447d-b71c-e2cfaeaee7cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1e7e2861-4689-447d-b71c-e2cfaeaee7cf\",\n \"name\": \"1e7e2861-4689-447d-b71c-e2cfaeaee7cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:17:56.829077997Z\",\n \"endTime\": \"2026-09-28T07:47:03.158495671Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint api-int.t0n9t0u4r4r5j6y.biea.j4259200.hcp.osadev.cloud failed: Get \\\"https://api-int.t0n9t0u4r4r5j6y.biea.j4259200.hcp.osadev.cloud:443/healthz?verbose\\\": context deadline exceeded (Client.Timeout exceeded while awaiting headers); hosted cluster degraded: UnavailableReplicas: redhat-operators-catalog deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c073278b4df6afb8cf91007966cde1fa7c3453fd0ebf082892677572d0b03f07","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: health check to APIServer endpoint \u003chcp-api-host\u003e failed: Get \"\u003curl\u003e\": context deadline exceeded (Client.Timeout exceeded while awaiting headers); hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"2849c4480c2b20826a373408395067a2ca4b1c46f8394bdf7a7d7fb6a54f1ca8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"e2e","signature_id":"dca12c8dd7fe8182c1727d2cbd5b05ff33a4efdd1526d2cfc5c44292de098c00","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019421c8\u003e: \n resource group \"rg-candidate-5-0-z7h95fzkng4w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-tvrknh\" in resourcegroup=\"rg-candidate-5-0-z7h95fzkng4w\" to finish deleting: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/acf5d502-002a-40c9-939d-0b196b8376ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/acf5d502-002a-40c9-939d-0b196b8376ff\",\n \"name\": \"acf5d502-002a-40c9-939d-0b196b8376ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:42:39.657365818Z\",\n \"endTime\": \"2026-09-28T08:06:43.09244758Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3sd9jqifdo5nbri2gfug3fjubjauaa still exists (deletion dispatched at 2026-09-28T07:42:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019421c8\u003e: \n resource group \"rg-candidate-5-0-z7h95fzkng4w\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-5-0-tvrknh\" in resourcegroup=\"rg-candidate-5-0-z7h95fzkng4w\" to finish deleting: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/acf5d502-002a-40c9-939d-0b196b8376ff\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/acf5d502-002a-40c9-939d-0b196b8376ff\",\n \"name\": \"acf5d502-002a-40c9-939d-0b196b8376ff\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:42:39.657365818Z\",\n \"endTime\": \"2026-09-28T08:06:43.09244758Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3sd9jqifdo5nbri2gfug3fjubjauaa still exists (deletion dispatched at 2026-09-28T07:42:46Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2d2a996a81608111c74cf84838ce844f0445feab2a1f2b3e9c43f07091af16d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"alert","signature_id":"df04e28ceb24729f941b045e921b610aab82e4c5f836a237e9b786cc3ade9993","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T07:32:17Z\n Ended: 2026-09-28T08:32:17Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3sd8fa4sfo5kgkevh0op6a88e9j5ei-z7j8g5y8a5a3b8k\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3sd8fa4sfo5kgkevh0op6a88e9j5ei-z7j8g5y8a5a3b8k\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T07:35:16Z\n Ended: 2026-09-28T08:07:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3scp4va5n5nn2ra64ruq0e3s68195j-u1z0s4v7m7d0a6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3scp4va5n5nn2ra64ruq0e3s68195j-u1z0s4v7m7d0a6h\n\nFiring 3:\n State: Resolved\n Started: 2026-09-28T07:37:17Z\n Ended: 2026-09-28T08:07:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4259200-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3sdcaqi76dbq451pj95v1uqnfr208u-ingress-cert-le\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4259200-mgmt-2\nNamespace: ocm-arohcpci01-2t3sdcaqi76dbq451pj95v1uqnfr208u-ingress-cert-le\n\nFiring 4:\n State: Resolved\n Started: 2026-09-28T07:37:17Z\n Ended: 2026-09-28T07:57:16Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3sd354a3bjr7bte08phb8vflqovkof-acr-pull-create\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3sd354a3bjr7bte08phb8vflqovkof-acr-pull-create\n\nFiring 5:\n State: Resolved\n Started: 2026-09-28T07:43:19Z\n Ended: 2026-09-28T08:13:17Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4259200-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4259200-mgmt-2\nNamespace: ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"2d9d20fc31e47ac001f94f5d5020121e91577e382bebaa10f7d7fc68aa1ad747","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"e2e","signature_id":"8f9fa274992a586c59ef9b66ef85d395e00eb0397fa57b1ab28990ac84af7438","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001138408\u003e: \n resource group \"rg-candidate-4-23-thxgh9fqwrb4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-8jxpm2\" in resourcegroup=\"rg-candidate-4-23-thxgh9fqwrb4\" to finish deleting: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/140bfc0f-802a-403b-ad5f-d4afa52fb8f7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/140bfc0f-802a-403b-ad5f-d4afa52fb8f7\",\n \"name\": \"140bfc0f-802a-403b-ad5f-d4afa52fb8f7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:40:12.927522807Z\",\n \"endTime\": \"2026-09-28T08:04:13.132621699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3sd9i7p3emdffa2gd3ri3elbvmi10h still exists (deletion dispatched at 2026-09-28T07:40:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001138408\u003e: \n resource group \"rg-candidate-4-23-thxgh9fqwrb4\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-8jxpm2\" in resourcegroup=\"rg-candidate-4-23-thxgh9fqwrb4\" to finish deleting: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/140bfc0f-802a-403b-ad5f-d4afa52fb8f7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/140bfc0f-802a-403b-ad5f-d4afa52fb8f7\",\n \"name\": \"140bfc0f-802a-403b-ad5f-d4afa52fb8f7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:40:12.927522807Z\",\n \"endTime\": \"2026-09-28T08:04:13.132621699Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3sd9i7p3emdffa2gd3ri3elbvmi10h still exists (deletion dispatched at 2026-09-28T07:40:14Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"38418ae9ee46d4311fc5f071f155b54ea1ffcf92356b0159ccb1ed413ca7c1b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"e2e","signature_id":"168ed1ed23c3b84e54b2f0e49b128524a1e37906a697a2fe43f0b3969bdf66ae","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017a4198\u003e: \n resource group \"private-kas-c65n2fbgmgnm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-c65n2fbgmgnm\" to finish deleting: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d639e17-fc2d-4fbd-b7d8-4049afd23792\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d639e17-fc2d-4fbd-b7d8-4049afd23792\",\n \"name\": \"8d639e17-fc2d-4fbd-b7d8-4049afd23792\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:41:12.899620144Z\",\n \"endTime\": \"2026-09-28T08:05:13.036001959Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3sct197en3e7rl4tferai9as2qo90u still exists (deletion dispatched at 2026-09-28T07:41:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0017a4198\u003e: \n resource group \"private-kas-c65n2fbgmgnm\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kas\" in resourcegroup=\"private-kas-c65n2fbgmgnm\" to finish deleting: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d639e17-fc2d-4fbd-b7d8-4049afd23792\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/8d639e17-fc2d-4fbd-b7d8-4049afd23792\",\n \"name\": \"8d639e17-fc2d-4fbd-b7d8-4049afd23792\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:41:12.899620144Z\",\n \"endTime\": \"2026-09-28T08:05:13.036001959Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3sct197en3e7rl4tferai9as2qo90u still exists (deletion dispatched at 2026-09-28T07:41:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"58a3f08b8439f8963a34337efc60e2ce4dc4f686a01ec207bacf01af4a242774","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"alert","signature_id":"11ffe7254f8c26fa108e93294ba03af128f400124607c21f497d84718b3c6125","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T07:30:18Z\n Ended: 2026-09-28T07:54:19Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t\", pod=\"router-74979554f7-8crr2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t/router-74979554f7-8crr2 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T07:31:16Z\n Ended: 2026-09-28T07:56:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster\", pod=\"router-7b9fff456-6vss4\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster/router-7b9fff456-6vss4 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-28T07:31:16Z\n Ended: 2026-09-28T07:56:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster\", pod=\"router-57cc58f4cd-7msjr\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster/router-57cc58f4cd-7msjr has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-28T07:33:18Z\n Ended: 2026-09-28T07:56:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4259200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r\", pod=\"router-7698497b4b-2px6r\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r/router-7698497b4b-2px6r has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-28T07:33:19Z\n Ended: 2026-09-28T07:56:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4259200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r\", pod=\"router-7698497b4b-87nkt\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r/router-7698497b4b-87nkt has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"6923e114ca6eea6e3e4c2a9c9bfc5fabc05bd26c0bad085ab8eea9e1303c28d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"alert","signature_id":"aef3898db187315c24458f77943698d78ec2e1a6b275d86eed3a6518338631b3","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T07:33:17Z\n Ended: 2026-09-28T07:42:17Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3scp4va5n5nn2ra64ruq0e3s68195j-u1z0s4v7m7d0a6h\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3scp4va5n5nn2ra64ruq0e3s68195j-u1z0s4v7m7d0a6h\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T07:34:21Z\n Ended: 2026-09-28T07:42:17Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j4259200-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3sdcaqi76dbq451pj95v1uqnfr208u-ingress-cert-le\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4259200-mgmt-2\nNamespace: ocm-arohcpci01-2t3sdcaqi76dbq451pj95v1uqnfr208u-ingress-cert-le\n\nFiring 3:\n State: Resolved\n Started: 2026-09-28T07:39:16Z\n Ended: 2026-09-28T07:49:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3sd15tp1vsqeia30g4p4aldq8urook-i5j0s3k4w4m1r6u\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3sd15tp1vsqeia30g4p4aldq8urook-i5j0s3k4w4m1r6u","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"6c5a84b117358851edd723efabe8739f59532832a5d5208aaf56ab2bc1fac449","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"e2e","signature_id":"0b89bdc7383f2c123c8287a1a5aec43b3c515722ab8e7d58f23bb0002aec1e93","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000911660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-6qt82pv5cb4k\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd024c78-818b-4b49-8634-2f704d9a8088\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd024c78-818b-4b49-8634-2f704d9a8088\",\n \"name\": \"cd024c78-818b-4b49-8634-2f704d9a8088\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:18:28.191649845Z\",\n \"endTime\": \"2026-09-28T07:47:33.272794005Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:99]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000911660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-6qt82pv5cb4k\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd024c78-818b-4b49-8634-2f704d9a8088\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/cd024c78-818b-4b49-8634-2f704d9a8088\",\n \"name\": \"cd024c78-818b-4b49-8634-2f704d9a8088\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:18:28.191649845Z\",\n \"endTime\": \"2026-09-28T07:47:33.272794005Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa7e67c7a590a1275d29fc48bfa890bb54cd4ebb44d206ef3a2ff048e5a51837","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"802c07f11e0f2629fc085177115227537ce4dfcd7692cfe3904197435e78548e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"e2e","signature_id":"560cf0a1b50bdf979e6f126cbc4861fee95b265a174c9f1c0ba41fcc25b22b09","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--hspw4fkzkwjz/cluster-hs-pre-nhlgdc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb3280\u003e: \n failed to create HCP cluster cluster-hs-pre-nhlgdc: failed waiting for cluster=\"cluster-hs-pre-nhlgdc\" in resourcegroup=\"rg-hs-pre--hspw4fkzkwjz\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e30d74a4-97b0-45b3-bb34-c0689a890bb7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e30d74a4-97b0-45b3-bb34-c0689a890bb7\",\n \"name\": \"e30d74a4-97b0-45b3-bb34-c0689a890bb7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:17:58.926212691Z\",\n \"endTime\": \"2026-09-28T07:47:03.131231516Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m5s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--hspw4fkzkwjz/cluster-hs-pre-nhlgdc should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb3280\u003e: \n failed to create HCP cluster cluster-hs-pre-nhlgdc: failed waiting for cluster=\"cluster-hs-pre-nhlgdc\" in resourcegroup=\"rg-hs-pre--hspw4fkzkwjz\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e30d74a4-97b0-45b3-bb34-c0689a890bb7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e30d74a4-97b0-45b3-bb34-c0689a890bb7\",\n \"name\": \"e30d74a4-97b0-45b3-bb34-c0689a890bb7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:17:58.926212691Z\",\n \"endTime\": \"2026-09-28T07:47:03.131231516Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m5s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9eb9c04adb0791b05386e2bc56953e6be0f9560103bd901c0d21c4e4506aca2f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"alert","signature_id":"84600d415d69723b878c3431a0471785536256c27ac395972895dceb4cac7a86","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T07:36:16Z\n Ended: 2026-09-28T08:07:18Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3scp4va5n5nn2ra64ruq0e3s68195j-u1z0s4v7m7d0a6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3scp4va5n5nn2ra64ruq0e3s68195j-u1z0s4v7m7d0a6h\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T07:37:17Z\n Ended: 2026-09-28T07:44:20Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j4259200-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3sdcaqi76dbq451pj95v1uqnfr208u-ingress-cert-le\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4259200-mgmt-2\nNamespace: ocm-arohcpci01-2t3sdcaqi76dbq451pj95v1uqnfr208u-ingress-cert-le\n\nFiring 3:\n State: Resolved\n Started: 2026-09-28T07:42:17Z\n Ended: 2026-09-28T08:14:17Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3sd15tp1vsqeia30g4p4aldq8urook-i5j0s3k4w4m1r6u\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3sd15tp1vsqeia30g4p4aldq8urook-i5j0s3k4w4m1r6u","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"ad2b624f8725915143ec549f5df1c873bf03114aa7d1b141e3e3c58810780917","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"e2e","signature_id":"8407ced857bcce3a5f161f49e7fb898405968e74316d0c49111a8c01a8f61c87","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b72f40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-pqfhb9gph6n6\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1554388-8ac1-4e37-a154-657d9e219a9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1554388-8ac1-4e37-a154-657d9e219a9c\",\n \"name\": \"e1554388-8ac1-4e37-a154-657d9e219a9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:33:16.16764174Z\",\n \"endTime\": \"2026-09-28T07:52:22.952806599Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-dl2qp-qdb4g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-pqfhb9gph6n6--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dl2qp-qdb4g to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-dl2qp-qdb4g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-pqfhb9gph6n6--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dl2qp-qdb4g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b72f40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-pqfhb9gph6n6\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1554388-8ac1-4e37-a154-657d9e219a9c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1554388-8ac1-4e37-a154-657d9e219a9c\",\n \"name\": \"e1554388-8ac1-4e37-a154-657d9e219a9c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:33:16.16764174Z\",\n \"endTime\": \"2026-09-28T07:52:22.952806599Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine v20261001-np-1-dl2qp-qdb4g: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-pqfhb9gph6n6--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dl2qp-qdb4g to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine v20261001-np-1-dl2qp-qdb4g: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-pqfhb9gph6n6--managed/providers/Microsoft.Compute/virtualMachines/v20261001-np-1-dl2qp-qdb4g to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b04c0d35ad368b694d61e29f9fae58554958243dce5a105b3482509e37755045","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"alert","signature_id":"8e9a5272f264a6a6fbd212995fbea44c0c88cab133bdbe6fddc3a31df7edd368","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T07:53:06Z\n Ended: 2026-09-28T07:59:07Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4259200-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4259200-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"d03fbea0c95ce406e9de15b7c66500cfa51c66a389844d6e5b3ea29335edc017","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"alert","signature_id":"ba7bab6629d6e8d805675d29c69fdec9a750497cd38dbfd826ca13c707567172","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T07:29:17Z\n Ended: 2026-09-28T08:07:16Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3sd8fa4sfo5kgkevh0op6a88e9j5ei-z7j8g5y8a5a3b8k\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3sd8fa4sfo5kgkevh0op6a88e9j5ei-z7j8g5y8a5a3b8k\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T07:32:17Z\n Ended: 2026-09-28T07:42:15Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3scp4va5n5nn2ra64ruq0e3s68195j-u1z0s4v7m7d0a6h\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3scp4va5n5nn2ra64ruq0e3s68195j-u1z0s4v7m7d0a6h\n\nFiring 3:\n State: Resolved\n Started: 2026-09-28T07:34:16Z\n Ended: 2026-09-28T07:48:15Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4259200-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3sd354a3bjr7bte08phb8vflqovkof-acr-pull-create\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4259200-mgmt-1\nNamespace: ocm-arohcpci01-2t3sd354a3bjr7bte08phb8vflqovkof-acr-pull-create\n\nFiring 4:\n State: Resolved\n Started: 2026-09-28T07:34:16Z\n Ended: 2026-09-28T07:44:16Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4259200-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3sdcaqi76dbq451pj95v1uqnfr208u-ingress-cert-le\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4259200-mgmt-2\nNamespace: ocm-arohcpci01-2t3sdcaqi76dbq451pj95v1uqnfr208u-ingress-cert-le\n\nFiring 5:\n State: Resolved\n Started: 2026-09-28T07:40:15Z\n Ended: 2026-09-28T07:50:15Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4259200-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4259200-mgmt-2\nNamespace: ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"e25d5cbe7fa90a45fd80b9f8300b0b3426696c4e5db219a226d31837247684e2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"e2e","signature_id":"39d100c30a8f93dbf6247cda0ea2c5258c578cc34be4d24eed8741b80b0be4ac","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015a0780\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5p54bqs5s2wq\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1573c01-1be7-4a7a-a184-70dcc6eb1000\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1573c01-1be7-4a7a-a184-70dcc6eb1000\",\n \"name\": \"b1573c01-1be7-4a7a-a184-70dcc6eb1000\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:18:48.290879335Z\",\n \"endTime\": \"2026-09-28T07:47:53.160718992Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015a0780\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-5p54bqs5s2wq\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1573c01-1be7-4a7a-a184-70dcc6eb1000\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/b1573c01-1be7-4a7a-a184-70dcc6eb1000\",\n \"name\": \"b1573c01-1be7-4a7a-a184-70dcc6eb1000\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:18:48.290879335Z\",\n \"endTime\": \"2026-09-28T07:47:53.160718992Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 23m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e3cfe1e30520077095f29aae791e72f185ecbe06141cabd583a9372f015ebd0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"alert","signature_id":"a5285cd524928261e15e6eab19f259927d80a331ee8b33927eebc0a295dec504","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 16 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-28T08:26:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-28T08:26:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-28T08:26:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-28T08:26:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-28T08:27:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-28T08:27:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-28T08:29:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.233:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z54j2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-09-28T08:29:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.233:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z54j2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-09-28T08:29:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.233:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z54j2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-09-28T08:29:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.233:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-z54j2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sdh60bmkrgtngq1o0rh0b4nlqksl7-d4l4a8z0v3y5r6r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-09-28T08:30:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Fired (not resolved)\n Started: 2026-09-28T08:30:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sdmetkr1pjassdmepvbt8femrrbda-cilium-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Fired (not resolved)\n Started: 2026-09-28T08:30:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Fired (not resolved)\n Started: 2026-09-28T08:30:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Fired (not resolved)\n Started: 2026-09-28T08:30:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3se1b6bje1ulsarhngi9asa53lodqr-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Fired (not resolved)\n Started: 2026-09-28T08:30:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4259200-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.78:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4259200/providers/microsoft.monitor/accounts/hcps-j4259200\", namespace=\"ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-f4z5m\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3sd9oqcm1kefsb7dqks9ar62agsjku-e7g4a1t9d1d9r8t/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"f123cb5f63f60c9e3fceccb224ff639c03e0128b273e01058219e5e1891d9f6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104457246334259200","occurred_at":"2026-09-28T06:24:41Z","failed_at":"e2e","signature_id":"e02b83b7b7d277f528654c1a7a21a3360f6f5084d7e91c17cd6c997d050b7856","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fda740\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dsp8rmtwf8ld\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70235f5c-fb03-427a-9ea4-ad4fabb543b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70235f5c-fb03-427a-9ea4-ad4fabb543b4\",\n \"name\": \"70235f5c-fb03-427a-9ea4-ad4fabb543b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:19:31.516544795Z\",\n \"endTime\": \"2026-09-28T07:48:33.353465776Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m12s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fda740\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-dsp8rmtwf8ld\" to finish creating: GET https://rp.j4259200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70235f5c-fb03-427a-9ea4-ad4fabb543b4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/70235f5c-fb03-427a-9ea4-ad4fabb543b4\",\n \"name\": \"70235f5c-fb03-427a-9ea4-ad4fabb543b4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T07:19:31.516544795Z\",\n \"endTime\": \"2026-09-28T07:48:33.353465776Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.35 is Partial (want Completed), started 24m12s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":16,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","aa7e67c7a590a1275d29fc48bfa890bb54cd4ebb44d206ef3a2ff048e5a51837","c073278b4df6afb8cf91007966cde1fa7c3453fd0ebf082892677572d0b03f07","d267405d540a4bece3363f10cc216deb4dc4b3fe44b839cd180b7e7fdfb1e54c","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":16,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441054601154560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7202,"pr_state":"open","pr_sha":"bf798191f1e4784ba8c222bbae084b193940b0ff","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T05:20:20Z","started_at":"2026-09-28T05:20:20Z","completed_at":"2026-09-28T07:49:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T07:53:14.688942774Z","timing_metadata_checked_at":"2026-09-28T07:53:14.688942774Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T07:53:14.692172764Z","region_metadata_checked_at":"2026-09-28T07:53:14.692172764Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"255dadef453877c4be4ed21b7a1d2610c024192dd4d431ed4c077a6c616d7994","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441054601154560","occurred_at":"2026-09-28T05:20:20Z","failed_at":"alert","signature_id":"0a61a6329e586b5af75e7ba37f30cc0d75552d5e500dee8165072e8b2848fbf4","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T06:58:33Z\n Ended: 2026-09-28T07:06:36Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1154560-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1154560-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"490b51eeaa7c5cf1da7919f75d65facca0e64d852ea02fa7008b080fbc9e4831","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441054601154560","occurred_at":"2026-09-28T05:20:20Z","failed_at":"alert","signature_id":"f7140b732d2b9ada8666aae03ac943606c09447431321d689e9e3f694f2a46d7","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T06:51:50Z\n Ended: 2026-09-28T06:57:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j1154560-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3rl3f8tj7aka16mnasqurrb5ebbvn0-w7g1w7z1p7g5a8x\", pod=\"kube-apiserver-8448d69f6f-jqnnb\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3rl3f8tj7aka16mnasqurrb5ebbvn0-w7g1w7z1p7g5a8x/kube-apiserver-8448d69f6f-jqnnb has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"bc12ec249f54e594ba3e1180c2c777d08d5169f19c124b690d7891c0b0c70a31","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441054601154560","occurred_at":"2026-09-28T05:20:20Z","failed_at":"e2e","signature_id":"f7628f13fed9510eb34469be7d0d678932294a06271b2a52d63d608fd70b72d1","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2f340\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rxccfwgxq7jj\" to finish creating: GET https://rp.j1154560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ba1066e-9913-4a3a-ab91-5b893d5447b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ba1066e-9913-4a3a-ab91-5b893d5447b0\",\n \"name\": \"7ba1066e-9913-4a3a-ab91-5b893d5447b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T06:32:25.841883321Z\",\n \"endTime\": \"2026-09-28T06:51:27.808098854Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b6o4b6r8b3p4s9g-np-1-s8xp9-hrvhd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-rxccfwgxq7jj--managed/b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rxccfwgxq7jj--managed/providers/Microsoft.Network/networkInterfaces/b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '3' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b2f340\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-rxccfwgxq7jj\" to finish creating: GET https://rp.j1154560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ba1066e-9913-4a3a-ab91-5b893d5447b0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/7ba1066e-9913-4a3a-ab91-5b893d5447b0\",\n \"name\": \"7ba1066e-9913-4a3a-ab91-5b893d5447b0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T06:32:25.841883321Z\",\n \"endTime\": \"2026-09-28T06:51:27.808098854Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b6o4b6r8b3p4s9g-np-1-s8xp9-hrvhd: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs: Failed: interfaces failed to create or update. err: failed to get existing resource fips-enabled-rxccfwgxq7jj--managed/b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-rxccfwgxq7jj--managed/providers/Microsoft.Network/networkInterfaces/b6o4b6r8b3p4s9g-np-1-s8xp9-fsxfs-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'b117cf0f-0276-4486-9691-4f00f5a90e1b' exceeded. Please try again after '3' seconds after additional tokens are available. Refer to http... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Network","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a30c60b06990338665e13dfa8bb11fe34f7411c3476ba7b78f153764bec29383","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441035131195392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7203,"pr_state":"open","pr_sha":"75d72389678ef3781f99923a095f0f5dfe0094f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T05:20:15Z","started_at":"2026-09-28T05:20:15Z","completed_at":"2026-09-28T07:50:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T08:03:56.55341937Z","timing_metadata_checked_at":"2026-09-28T08:03:56.55341937Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T08:03:56.556004262Z","region_metadata_checked_at":"2026-09-28T08:03:56.556004262Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"2800036250cb0cbca6f91e83583ce359e3e6b1addd791f0fa28e5cdfa7e6018d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441035131195392","occurred_at":"2026-09-28T05:20:15Z","failed_at":"alert","signature_id":"679e5de886d8c0615ef4790483cd7a9e61e288f3405b68c87e65af78de307ef6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T06:57:58Z\n Ended: 2026-09-28T07:09:57Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1195392-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j1195392-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441018513362944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T05:20:12Z","started_at":"2026-09-28T05:20:12Z","completed_at":"2026-09-28T07:42:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T07:53:14.247643336Z","timing_metadata_checked_at":"2026-09-28T07:53:14.247643336Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T07:53:14.250403228Z","region_metadata_checked_at":"2026-09-28T07:53:14.250403228Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c8d65b9eb1440e12d9d4a45c2095a56e80d822dbbb07c6685c45ad55e1ea21ae","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104441018513362944","occurred_at":"2026-09-28T05:20:12Z","failed_at":"e2e","signature_id":"5066cec9d4455e60a98c73837fba1de819802d265c19a56ae10aab27b4f06354","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001303da0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-n7hm7qphnd7k\" to finish creating: GET https://rp.j3362944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/14660fe0-ba9c-4568-a549-06eba5f1111c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/14660fe0-ba9c-4568-a549-06eba5f1111c\",\n \"name\": \"14660fe0-ba9c-4568-a549-06eba5f1111c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T06:35:59.37107386Z\",\n \"endTime\": \"2026-09-28T06:55:00.67957674Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-hhqxx-z5glq: Failed: virtualmachine failed to create or update. err: failed to get existing resource complex-cilium-kv-n7hm7qphnd7k--managed/cilium-cluster-cilium-np-hhqxx-z5glq (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-n7hm7qphnd7k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-hhqxx-z5glq\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are ava... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:205]: failed to verify nodes are Ready with Cilium CNI for cluster \"cilium-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001303da0\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-n7hm7qphnd7k\" to finish creating: GET https://rp.j3362944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/14660fe0-ba9c-4568-a549-06eba5f1111c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/14660fe0-ba9c-4568-a549-06eba5f1111c\",\n \"name\": \"14660fe0-ba9c-4568-a549-06eba5f1111c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T06:35:59.37107386Z\",\n \"endTime\": \"2026-09-28T06:55:00.67957674Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cluster-cilium-np-hhqxx-z5glq: Failed: virtualmachine failed to create or update. err: failed to get existing resource complex-cilium-kv-n7hm7qphnd7k--managed/cilium-cluster-cilium-np-hhqxx-z5glq (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/complex-cilium-kv-n7hm7qphnd7k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cluster-cilium-np-hhqxx-z5glq\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '3d8c36e1-ae7a-42ef-b7d1-ea4667708d30' exceeded. Please try again after '10' seconds after additional tokens are ava... message truncated\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440974754189312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T05:20:01Z","started_at":"2026-09-28T05:20:01Z","completed_at":"2026-09-28T07:48:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T07:53:13.933815905Z","timing_metadata_checked_at":"2026-09-28T07:53:13.933815905Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T07:53:13.937486193Z","region_metadata_checked_at":"2026-09-28T07:53:13.937486193Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"276fe8aea78a1df3f787042910337d17fe923264e20710ac8163c4b9c8a9fb85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440974754189312","occurred_at":"2026-09-28T05:20:01Z","failed_at":"e2e","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"9a7c51f322e7d62418865c5276de9259cd31bad135006aaa86c63177fd9c1103","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440974754189312","occurred_at":"2026-09-28T05:20:01Z","failed_at":"alert","signature_id":"10d54fdb3fd853f5743498ec14b91987dd77b2bc92079478e8da6f76b7e1d6bd","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T07:01:37Z\n Ended: 2026-09-28T07:07:34Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4189312-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4189312-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"c88a841cea2e3c3c25fec732e8c9ac419145867814372f3bd1317e2431f1e38c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440974754189312","occurred_at":"2026-09-28T05:20:01Z","failed_at":"e2e","signature_id":"ffaedd37a7cabdde88484871ed63e277dfa5e461f7bbb1806cffb14d8667ee33","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad61a0\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-nfzkdr\" in resourcegroup=\"rg-np-downgrade-nfzkdr-5bwdcqfbzpwz\" to finish creating: GET https://rp.j4189312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca540d48-e274-44eb-b0a7-ddf6f21af0cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca540d48-e274-44eb-b0a7-ddf6f21af0cc\",\n \"name\": \"ca540d48-e274-44eb-b0a7-ddf6f21af0cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T06:59:50.075602846Z\",\n \"endTime\": \"2026-09-28T07:18:52.768957656Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i2k1v9u6g0j4j1a-npdg-4-21-tbbgs-mxb9k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-nfzkdr-5bwdcqfbzpwz-np-dg-nfzkdr--managed/providers/Microsoft.Compute/virtualMachines/i2k1v9u6g0j4j1a-npdg-4-21-tbbgs-mxb9k to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i2k1v9u6g0j4j1a-npdg-4-21-tbbgs-mxb9k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-nfzkdr-5bwdcqfbzpwz-np-dg-nfzkdr--managed/providers/Microsoft.Compute/virtualMachines/i2k1v9u6g0j4j1a-npdg-4-21-tbbgs-mxb9k to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ad61a0\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-nfzkdr\" in resourcegroup=\"rg-np-downgrade-nfzkdr-5bwdcqfbzpwz\" to finish creating: GET https://rp.j4189312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca540d48-e274-44eb-b0a7-ddf6f21af0cc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ca540d48-e274-44eb-b0a7-ddf6f21af0cc\",\n \"name\": \"ca540d48-e274-44eb-b0a7-ddf6f21af0cc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T06:59:50.075602846Z\",\n \"endTime\": \"2026-09-28T07:18:52.768957656Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine i2k1v9u6g0j4j1a-npdg-4-21-tbbgs-mxb9k: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-nfzkdr-5bwdcqfbzpwz-np-dg-nfzkdr--managed/providers/Microsoft.Compute/virtualMachines/i2k1v9u6g0j4j1a-npdg-4-21-tbbgs-mxb9k to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine i2k1v9u6g0j4j1a-npdg-4-21-tbbgs-mxb9k: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-nfzkdr-5bwdcqfbzpwz-np-dg-nfzkdr--managed/providers/Microsoft.Compute/virtualMachines/i2k1v9u6g0j4j1a-npdg-4-21-tbbgs-mxb9k to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440964679471104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"closed","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T05:19:58Z","started_at":"2026-09-28T05:19:58Z","completed_at":"2026-09-28T05:45:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T05:55:42.526929899Z","timing_metadata_checked_at":"2026-09-28T05:55:42.526929899Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-28T05:55:42.529878189Z","region_metadata_checked_at":"2026-09-28T07:00:13.029127223Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"71e306a121efcea327612f20d50d7a326fab3d42cd2718dc8eeb45d161209d1a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440964679471104","occurred_at":"2026-09-28T05:19:58Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440930432978944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T05:19:51Z","started_at":"2026-09-28T05:19:51Z","completed_at":"2026-09-28T08:04:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T08:14:39.786818853Z","timing_metadata_checked_at":"2026-09-28T08:14:39.786818853Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T08:14:39.79057294Z","region_metadata_checked_at":"2026-09-28T08:14:39.79057294Z"},"failed_at":["alert"],"failed_test_count":3,"occurrences":[{"row_id":"42a025b86ad838ceb10afaa82a29d5c5043a3c5e1434ac7c190e78d312ec44a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440930432978944","occurred_at":"2026-09-28T05:19:51Z","failed_at":"alert","signature_id":"5eb0f55666a6428c416d4036a93a530ef71389fa1bd3105e20ed8c531ec7abad","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T06:58:35Z\n Ended: 2026-09-28T07:06:36Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2978944-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2978944-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"d6cbd0ba97b7da96c50c14c57ad89210279ed94956f4944e4bb2501c04d1c228","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440930432978944","occurred_at":"2026-09-28T05:19:51Z","failed_at":"alert","signature_id":"6b2918236ea1d95b7e6826a1ce35aa62303eaac8605953f4e310050355cbd27e","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T06:52:56Z\n Ended: 2026-09-28T07:21:54Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2978944-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3riui9072n9eok39gpkgl46upeo7m9-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2978944-mgmt-2\nNamespace: ocm-arohcpci01-2t3riui9072n9eok39gpkgl46upeo7m9-private-kas\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T07:37:54Z\n Ended: 2026-09-28T07:45:54Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2978944-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3rsi6td6p6c241sbjaqid23velc7j6-j6u3g7r8x8f0v5c\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2978944-mgmt-1\nNamespace: ocm-arohcpci01-2t3rsi6td6p6c241sbjaqid23velc7j6-j6u3g7r8x8f0v5c","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"f773cf8a8143bf47f81aa590166637735977f5bd43beaf4607246d5ae5fdd9ab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104440930432978944","occurred_at":"2026-09-28T05:19:51Z","failed_at":"alert","signature_id":"10f839c05a115a040846a911b08d87f6a5ef0e8b18e1acd188a5d746ed43ef16","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T06:49:57Z\n Ended: 2026-09-28T06:56:55Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2978944-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3riui9072n9eok39gpkgl46upeo7m9-private-kas\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2978944-mgmt-2\nNamespace: ocm-arohcpci01-2t3riui9072n9eok39gpkgl46upeo7m9-private-kas\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T07:34:53Z\n Ended: 2026-09-28T07:42:55Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2978944-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3rsi6td6p6c241sbjaqid23velc7j6-j6u3g7r8x8f0v5c\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2978944-mgmt-1\nNamespace: ocm-arohcpci01-2t3rsi6td6p6c241sbjaqid23velc7j6-j6u3g7r8x8f0v5c","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104433568867422208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6869,"pr_state":"open","pr_sha":"4058f9eba8fc7b0fb6338eb0433682c6448de89d","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T04:50:35Z","started_at":"2026-09-28T04:50:35Z","completed_at":"2026-09-28T07:01:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T07:10:51.53180259Z","timing_metadata_checked_at":"2026-09-28T07:10:51.53180259Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T07:10:51.53498828Z","region_metadata_checked_at":"2026-09-28T07:10:51.53498828Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"9197608356414b489469c244d6e9841d54d0f49f571a74397d5e95531a87f1b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104433568867422208","occurred_at":"2026-09-28T04:50:35Z","failed_at":"alert","signature_id":"311fd8960a4e8d6a6edebec933ada94dd10107fdfe37f5ec1b9508c64913d1bd","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T06:12:52Z\n Ended: 2026-09-28T06:19:52Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7422208-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7422208-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"ce6c5aeb1bb2ecf00cb3e88b1b24b5376711f6c5c6dfff76b59f20c548d31da5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6869/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104433568867422208","occurred_at":"2026-09-28T04:50:35Z","failed_at":"e2e","signature_id":"cd7082ea03cbb08c0e4b8bdec829e02f4c6a4f777acb3eb5db34925ed981a912","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-x7z7b7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bab020\u003e: \n DNS for route host agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:116]: failed to verify simple web app runs on cluster \"cluster-hs-pre-x7z7b7\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bab020\u003e: \n DNS for route host agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104429033809776640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6226,"pr_state":"closed","pr_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","final_merged_sha":"9b50264285a60d3666e3df261a54ae027b4564e9","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T04:32:34Z","started_at":"2026-09-28T04:32:34Z","completed_at":"2026-09-28T06:43:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T06:49:16.371090614Z","timing_metadata_checked_at":"2026-09-28T06:49:16.371090614Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T06:49:16.377753793Z","region_metadata_checked_at":"2026-09-28T06:49:16.377753793Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"029d3c26b51bc207120914282e86d440e6e746d04f9e33abfc734ae3ef3e15b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104429033809776640","occurred_at":"2026-09-28T04:32:34Z","failed_at":"e2e","signature_id":"04c64a39622e30f071bc799f77f8bc41c83ad2872a342f4b19cf528cbde3c282","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000874f60\u003e: \n failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:219]: failed to look up CAPZ principal ID\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000874f60\u003e: \n failed to get CAPZ MI: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-dev-shard3-00-14/providers/Microsoft.ManagedIdentity/userAssignedIdentities/cluster-api-azure\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: 429\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"429\",\n \"message\": \"Request is temporarily throttled because subscription XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX has issued too many requests. Retry after 1 seconds.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","failure_pattern":"ERROR CODE: 429; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: 429"}},{"row_id":"4e974f73923f12dae919d06b9e6459ba556706f17664a988b118fb0f31074251","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104429033809776640","occurred_at":"2026-09-28T04:32:34Z","failed_at":"e2e","signature_id":"0b9020620144e8fdaa405e64fc19ea1a76e527f3246863311db8886e6e3a419c","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c3c580\u003e: \n DNS for route host agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:158]: failed to verify simple web app runs on v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c3c580\u003e: \n DNS for route host agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-vqp78.apps.aro.v20261001.jzew.j9776640.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvf99.apps.aro.t2p4s2i1j4x6p8f.mtn4.j7422208.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"6d1fb7930e9bc90e545e99aa4e05f334c1203cdf6fbd00e1ffeca6f34ce1cd34","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104429033809776640","occurred_at":"2026-09-28T04:32:34Z","failed_at":"e2e","signature_id":"f24501ad0818be8a83c15e2425a5d1f36053df6413c3f821edd665347627ddd5","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf1400\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-9wpvql\" in resourcegroup=\"rg-np-version-upgrade-9wpvql-69cnxf97b854\" to finish creating: GET https://rp.j9776640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5e618a3-8d2b-42aa-9601-b69b45f47871\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5e618a3-8d2b-42aa-9601-b69b45f47871\",\n \"name\": \"a5e618a3-8d2b-42aa-9601-b69b45f47871\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T05:29:42.522112187Z\",\n \"endTime\": \"2026-09-28T05:48:46.557258592Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n1h7x2o1d8f9m4k-npupgrade-4-20-chfrz-jbjzj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9wpvql-69cnxf97b854-np-upgrade-9w-04ede2a4/providers/Microsoft.Compute/virtualMachines/n1h7x2o1d8f9m4k-npupgrade-4-20-chfrz-jbjzj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n1h7x2o1d8f9m4k-npupgrade-4-20-chfrz-jbjzj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9wpvql-69cnxf97b854-np-upgrade-9w-04ede2a4/providers/Microsoft.Compute/virtualMachines/n1h7x2o1d8f9m4k-npupgrade-4-20-chfrz-jbjzj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf1400\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-9wpvql\" in resourcegroup=\"rg-np-version-upgrade-9wpvql-69cnxf97b854\" to finish creating: GET https://rp.j9776640.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5e618a3-8d2b-42aa-9601-b69b45f47871\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/a5e618a3-8d2b-42aa-9601-b69b45f47871\",\n \"name\": \"a5e618a3-8d2b-42aa-9601-b69b45f47871\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T05:29:42.522112187Z\",\n \"endTime\": \"2026-09-28T05:48:46.557258592Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine n1h7x2o1d8f9m4k-npupgrade-4-20-chfrz-jbjzj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9wpvql-69cnxf97b854-np-upgrade-9w-04ede2a4/providers/Microsoft.Compute/virtualMachines/n1h7x2o1d8f9m4k-npupgrade-4-20-chfrz-jbjzj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine n1h7x2o1d8f9m4k-npupgrade-4-20-chfrz-jbjzj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-9wpvql-69cnxf97b854-np-upgrade-9w-04ede2a4/providers/Microsoft.Compute/virtualMachines/n1h7x2o1d8f9m4k-npupgrade-4-20-chfrz-jbjzj to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c990c4bd77c08247589506307d030d2d5a99455f851e2bcc169544153e125ede","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104429033809776640","occurred_at":"2026-09-28T04:32:34Z","failed_at":"alert","signature_id":"e707533b83464f0017c7724f1f2fa89bd9f8f29f5f20020a54be5188c8c5bf45","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T05:53:31Z\n Ended: 2026-09-28T05:59:31Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9776640-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9776640-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","9205c69c72a57356c2890f3b58a265a6f5459948112af87f37384e641208f260","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"942d17a70a130ecc74fc8c2a9727ab849836bd52","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T04:00:44Z","started_at":"2026-09-28T04:00:44Z","completed_at":"2026-09-28T06:20:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T06:28:18.642116523Z","timing_metadata_checked_at":"2026-09-28T06:28:18.642116523Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T06:28:18.644908914Z","region_metadata_checked_at":"2026-09-28T06:28:18.644908914Z"},"failed_at":["alert","e2e"],"failed_test_count":23,"occurrences":[{"row_id":"03c8a0e5379c776c0b2fe03fdc8dfd8433004595652fa00b4faa75b2431d1d52","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"927ea0a3feb6d2e147cb24c7843e4021b13aea7e7d34e9de65f81c89fd4479e6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f902d0\u003e: \n resource group \"rg-candidate-4-23-9gw25wvwtcq8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-h9898s\" in resourcegroup=\"rg-candidate-4-23-9gw25wvwtcq8\" to finish deleting: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f64f2b5-a8be-4840-8a0f-0ab62f2387b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f64f2b5-a8be-4840-8a0f-0ab62f2387b2\",\n \"name\": \"9f64f2b5-a8be-4840-8a0f-0ab62f2387b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T05:19:21.358341405Z\",\n \"endTime\": \"2026-09-28T05:43:24.1869414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3q9mk4rs0jgf269qvh2civ1ghcvp1q still exists (deletion dispatched at 2026-09-28T05:19:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001f902d0\u003e: \n resource group \"rg-candidate-4-23-9gw25wvwtcq8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-23-h9898s\" in resourcegroup=\"rg-candidate-4-23-9gw25wvwtcq8\" to finish deleting: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f64f2b5-a8be-4840-8a0f-0ab62f2387b2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9f64f2b5-a8be-4840-8a0f-0ab62f2387b2\",\n \"name\": \"9f64f2b5-a8be-4840-8a0f-0ab62f2387b2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T05:19:21.358341405Z\",\n \"endTime\": \"2026-09-28T05:43:24.1869414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3q9mk4rs0jgf269qvh2civ1ghcvp1q still exists (deletion dispatched at 2026-09-28T05:19:29Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2943e3e140e26b402842f9d93b6fa6fe2bb64000e7df8d1e1ba63d461855d060","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"d77acb6f3606c90c5b93fb7307e047cf9fa16eec87a5e89c1c9e889994c91696","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:164 with:\nexpected HTTP 200 from sample app via internal LB, got 503\nExpected\n \u003cstring\u003e: ...\nto equal\n \u003cstring\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 600.000s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:164 with:\nexpected HTTP 200 from sample app via internal LB, got 503\nExpected\n \u003cstring\u003e: ...\nto equal\n \u003cstring\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"08ad18e89c33b55d4b3e771dd8d81006f379c5497ad8187cd5d751dd18c0ebc4","failure_pattern":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal: Timed out after \u003cduration\u003es.","search_query":"Timed out after 600.000s."}},{"row_id":"42f7ddc7614291fab8509aae760282e411f1844a39c7d183211dda71067dbb22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"dc5aba529688a7c5d0344c6021726d3af95020d72a42d1fb5deae657451b585d","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc65a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-6vnwh8vjnz8r\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b3163de-a897-4f45-9b77-e785bdfab09f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b3163de-a897-4f45-9b77-e785bdfab09f\",\n \"name\": \"0b3163de-a897-4f45-9b77-e785bdfab09f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:53:53.798033495Z\",\n \"endTime\": \"2026-09-28T05:22:54.610455796Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:417]: failed to create HCP cluster \"sre-hcp-cluster-bootdiag\" for boot diagnostics test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc65a0\u003e: \n failed to create HCP cluster sre-hcp-cluster-bootdiag: failed waiting for cluster=\"sre-hcp-cluster-bootdiag\" in resourcegroup=\"admin-api-serialconsole-bootdiag-6vnwh8vjnz8r\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b3163de-a897-4f45-9b77-e785bdfab09f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0b3163de-a897-4f45-9b77-e785bdfab09f\",\n \"name\": \"0b3163de-a897-4f45-9b77-e785bdfab09f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:53:53.798033495Z\",\n \"endTime\": \"2026-09-28T05:22:54.610455796Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"257b448a58141006c11f51362759eea79da6b0398afa6beef61e4326de15f4d6","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"4546ae5af999994962b958e806bfd9d71511adbb7974a45885a9d353e515d20f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"59d450dd267abc9e54a533d56b6c387e30b1b1a9dc2279ecaa1ac415c1887478","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae31a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5cbtfzkp497x\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6576461c-4ae7-4100-90f9-94b3c7ae31de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6576461c-4ae7-4100-90f9-94b3c7ae31de\",\n \"name\": \"6576461c-4ae7-4100-90f9-94b3c7ae31de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:53:33.784163345Z\",\n \"endTime\": \"2026-09-28T05:22:34.354425424Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: failed to create HCP cluster \"hcp-cluster-np-128\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae31a0\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-5cbtfzkp497x\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6576461c-4ae7-4100-90f9-94b3c7ae31de\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6576461c-4ae7-4100-90f9-94b3c7ae31de\",\n \"name\": \"6576461c-4ae7-4100-90f9-94b3c7ae31de\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:53:33.784163345Z\",\n \"endTime\": \"2026-09-28T05:22:34.354425424Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog, certified-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b586f8fea86ba211104459fd89bb5ef2ea79917b36b639db22bd6e7bf973221c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, community-operators-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5cc9c59b9278cf49fa7c088aeac6397a5139641e002d76e02bc8ad733853eaa5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"772c0837ffbd02ac36ebd76dd5378bc1ffed78fbbc057c3ba94ac4fd6bb2e9af","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cb0840\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rbwvmrnzw8j5\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5bdae3db-e612-4f27-8693-02200538e08e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5bdae3db-e612-4f27-8693-02200538e08e\",\n \"name\": \"5bdae3db-e612-4f27-8693-02200538e08e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:23.74994885Z\",\n \"endTime\": \"2026-09-28T05:23:24.026384001Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cb0840\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-rbwvmrnzw8j5\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5bdae3db-e612-4f27-8693-02200538e08e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5bdae3db-e612-4f27-8693-02200538e08e\",\n \"name\": \"5bdae3db-e612-4f27-8693-02200538e08e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:23.74994885Z\",\n \"endTime\": \"2026-09-28T05:23:24.026384001Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, community-operators-catalog; hosted cluster degraded: UnavailableReplicas: [community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"3b936310d5449729e7fddac902aac9547b4c9f95ec0132ddb43eddb70da02556","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [community-operators-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5ddaf0ea4bc4c33790767a42b5c524c47a1ad611c3ef09a94319b392f272f300","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"dcbe953ab01aef145b8f85e06d22bffa5a686320c14b765bf9e95bf552ee460e","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005d2ce0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-sbtnmq6dk9ch\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f13de10-f443-42eb-b391-6417a9194758\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f13de10-f443-42eb-b391-6417a9194758\",\n \"name\": \"1f13de10-f443-42eb-b391-6417a9194758\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:55:46.663436186Z\",\n \"endTime\": \"2026-09-28T05:24:54.296612042Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:208]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005d2ce0\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-sbtnmq6dk9ch\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f13de10-f443-42eb-b391-6417a9194758\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1f13de10-f443-42eb-b391-6417a9194758\",\n \"name\": \"1f13de10-f443-42eb-b391-6417a9194758\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:55:46.663436186Z\",\n \"endTime\": \"2026-09-28T05:24:54.296612042Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa7e67c7a590a1275d29fc48bfa890bb54cd4ebb44d206ef3a2ff048e5a51837","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"5de7e2d6345195a681789f929846c398edc2e77bcf645e9380836dfada17d6c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"f7515d3eb405f48e343776d3361bb29d0c59e96063451c01a874bff6974c8467","test_name":"Customer should be able to create a cluster with ACR pull via managed identity and pull from a private ACR","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c41e0\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-jfwb2dgmj26t\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\",\n \"name\": \"a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:17.858913168Z\",\n \"endTime\": \"2026-09-28T05:23:24.118546984Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_container_registry.go:237]: failed to create HCP cluster with containerRegistry set\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010c41e0\u003e: \n failed waiting for cluster=\"acr-pull-create\" in resourcegroup=\"acr-pull-create-jfwb2dgmj26t\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\",\n \"name\": \"a6a2f8c2-ae4c-4f51-8a9f-44a84ab3a613\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:17.858913168Z\",\n \"endTime\": \"2026-09-28T05:23:24.118546984Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [catalog-operator deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0dd45aef10b8391c9e5053879220fb6ef98b320a9e44f86e49414d35d0aff7ee","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [catalog-operator, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"76d26ab9e00d62b9c140b088d8e622d04adaa4ed77a8c07baeaf5a5f5cd961d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"alert","signature_id":"c0ba742d3b8df7101a372c1ade1a76e79f1270d7ffa16216e6dc2f557ede0a32","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-28T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/hcps-j4374016\", namespace=\"ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-28T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/hcps-j4374016\", namespace=\"ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-28T06:02:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/hcps-j4374016\", namespace=\"ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-28T06:03:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/hcps-j4374016\", namespace=\"ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-28T06:03:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/hcps-j4374016\", namespace=\"ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-28T06:04:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/hcps-j4374016\", namespace=\"ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-28T06:06:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/hcps-j4374016\", namespace=\"ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-09-28T06:06:33Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/hcps-j4374016\", namespace=\"ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"81527fd8135992b117fe32d387b8a73085d67c93ee05cbb600cdaa7c7d99cbe7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"2a44402a154f3d167466daa8cbf44d7db60e9fcc2662826bf741581bf798c662","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0004e23d8\u003e: \n resource group \"private-keyvault-7rld8zcz29zp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7rld8zcz29zp\" to finish deleting: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4041230b-c16b-483c-b3c9-78fc4b2a05c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4041230b-c16b-483c-b3c9-78fc4b2a05c6\",\n \"name\": \"4041230b-c16b-483c-b3c9-78fc4b2a05c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T05:16:11.06337557Z\",\n \"endTime\": \"2026-09-28T05:40:14.206723479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3qacjj77p1dno3e8ufhl76fug7bvbc still exists (deletion dispatched at 2026-09-28T05:16:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0004e23d8\u003e: \n resource group \"private-keyvault-7rld8zcz29zp\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-7rld8zcz29zp\" to finish deleting: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4041230b-c16b-483c-b3c9-78fc4b2a05c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4041230b-c16b-483c-b3c9-78fc4b2a05c6\",\n \"name\": \"4041230b-c16b-483c-b3c9-78fc4b2a05c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T05:16:11.06337557Z\",\n \"endTime\": \"2026-09-28T05:40:14.206723479Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3qacjj77p1dno3e8ufhl76fug7bvbc still exists (deletion dispatched at 2026-09-28T05:16:12Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"85eb4be07b847dbe044915160ef4e9f246c6379331ca80041f66ef5153ff0de2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"65926fb3ef21067b7449af3c8bad1b64cbed97824fcac64f72df97d6b9cae9eb","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-2s722b\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bfeb60\u003e: \n failed to create HCP cluster np-noedge-2s722b: failed waiting for cluster=\"np-noedge-2s722b\" in resourcegroup=\"rg-np-noedge-2s722b-ttwnzxstsj4p\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18b3752b-dfe7-413c-83e0-e4570361ec93\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18b3752b-dfe7-413c-83e0-e4570361ec93\",\n \"name\": \"18b3752b-dfe7-413c-83e0-e4570361ec93\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:09.735885483Z\",\n \"endTime\": \"2026-09-28T05:23:14.561838226Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-2s722b\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bfeb60\u003e: \n failed to create HCP cluster np-noedge-2s722b: failed waiting for cluster=\"np-noedge-2s722b\" in resourcegroup=\"rg-np-noedge-2s722b-ttwnzxstsj4p\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18b3752b-dfe7-413c-83e0-e4570361ec93\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/18b3752b-dfe7-413c-83e0-e4570361ec93\",\n \"name\": \"18b3752b-dfe7-413c-83e0-e4570361ec93\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:09.735885483Z\",\n \"endTime\": \"2026-09-28T05:23:14.561838226Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"257b448a58141006c11f51362759eea79da6b0398afa6beef61e4326de15f4d6","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, redhat-operators-catalog]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"923e2649f017a0e4020f0eec1b169d9200b08207b80794330e95461eef6c85df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"alert","signature_id":"0fe28f25b5227c777c5fca5ce997d516e287587c33aeacd91934a6120a38de47","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T05:01:43Z\n Ended: 2026-09-28T05:34:42Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4374016-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4374016-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"9afbdf1899053e36e6a3427ca80cbfe4eee948fc1a49408e20c0534c9ea13fda","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"alert","signature_id":"aa9f4214ab7044d7b9bcd462877e240cc060062645e93bcbaa17696de3b6d418","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T05:05:43Z\n Ended: 2026-09-28T05:33:44Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4374016-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3q9oqqnica740m7cbrv4g0ui9okfrv-v20261001\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4374016-mgmt-2\nNamespace: ocm-arohcpci01-2t3q9oqqnica740m7cbrv4g0ui9okfrv-v20261001\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T05:08:45Z\n Ended: 2026-09-28T05:34:44Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4374016-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4374016-mgmt-1\nNamespace: ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y\n\nFiring 3:\n State: Resolved\n Started: 2026-09-28T05:09:44Z\n Ended: 2026-09-28T05:15:43Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4374016-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3q9v1bm9djo4ed6huo3q8b2cid8e8d-s4a1j7z9i0s2s7m\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4374016-mgmt-1\nNamespace: ocm-arohcpci01-2t3q9v1bm9djo4ed6huo3q8b2cid8e8d-s4a1j7z9i0s2s7m\n\nFiring 4:\n State: Resolved\n Started: 2026-09-28T05:11:44Z\n Ended: 2026-09-28T05:19:44Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4374016-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3q9ljvg6g8v7atf8e4uj1uvv1o29vp-z3b4r7r7o7d8c5l\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4374016-mgmt-2\nNamespace: ocm-arohcpci01-2t3q9ljvg6g8v7atf8e4uj1uvv1o29vp-z3b4r7r7o7d8c5l\n\nFiring 5:\n State: Resolved\n Started: 2026-09-28T05:12:46Z\n Ended: 2026-09-28T05:22:43Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4374016-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3q9md15na81g26g32hk4dt01vu985b-private-kas\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4374016-mgmt-1\nNamespace: ocm-arohcpci01-2t3q9md15na81g26g32hk4dt01vu985b-private-kas\n\nFiring 6:\n State: Resolved\n Started: 2026-09-28T05:17:42Z\n Ended: 2026-09-28T05:31:46Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j4374016-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3qa7kalf0nmi953o4ju9ljktm9ed87-i1j4h0b1t2z2z3r\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4374016-mgmt-2\nNamespace: ocm-arohcpci01-2t3qa7kalf0nmi953o4ju9ljktm9ed87-i1j4h0b1t2z2z3r","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"9b908db65a2c54e2d65faca73e4ad78565d59e8ec77e925e67035cf4dc79033b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"96243ae66c6d39c7d59fea442c69e34bfc8d95860eb89e28f5e0f3d2d01c561b","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106c320\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-qdqkz9rfmldx\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c723202-68f5-41c7-a5d4-762157d31d23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c723202-68f5-41c7-a5d4-762157d31d23\",\n \"name\": \"0c723202-68f5-41c7-a5d4-762157d31d23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:53:53.997312592Z\",\n \"endTime\": \"2026-09-28T05:22:54.143428969Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:89]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106c320\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-qdqkz9rfmldx\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c723202-68f5-41c7-a5d4-762157d31d23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0c723202-68f5-41c7-a5d4-762157d31d23\",\n \"name\": \"0c723202-68f5-41c7-a5d4-762157d31d23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:53:53.997312592Z\",\n \"endTime\": \"2026-09-28T05:22:54.143428969Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: [kube-apiserver deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"847f58c2e694684da512b86721767988d4eb7fbb0f488bd687e65af5d9a8f912","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a5ca3c86d39842d5786fd0eb535a8858753e50cdc21d49a6622bbc7dfe05e068","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"589125c66cd0fbbb7e8b00aa0c8a7cb91a2f86a1961306a34f41a9f722db5222","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--6kj9nw5mjrc2/cluster-hs-pre-fvbpxs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db85e0\u003e: \n failed to create HCP cluster cluster-hs-pre-fvbpxs: failed waiting for cluster=\"cluster-hs-pre-fvbpxs\" in resourcegroup=\"rg-hs-pre--6kj9nw5mjrc2\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f78ab67-468f-449c-890b-25423489ffcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f78ab67-468f-449c-890b-25423489ffcd\",\n \"name\": \"2f78ab67-468f-449c-890b-25423489ffcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:03.674200437Z\",\n \"endTime\": \"2026-09-28T05:23:04.406727689Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:83]: HCP cluster rg-hs-pre--6kj9nw5mjrc2/cluster-hs-pre-fvbpxs should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db85e0\u003e: \n failed to create HCP cluster cluster-hs-pre-fvbpxs: failed waiting for cluster=\"cluster-hs-pre-fvbpxs\" in resourcegroup=\"rg-hs-pre--6kj9nw5mjrc2\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f78ab67-468f-449c-890b-25423489ffcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f78ab67-468f-449c-890b-25423489ffcd\",\n \"name\": \"2f78ab67-468f-449c-890b-25423489ffcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:03.674200437Z\",\n \"endTime\": \"2026-09-28T05:23:04.406727689Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: community-operators-catalog, certified-operators-catalog, redhat-operators-catalog; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog deployment has 1 unavailable replicas, community-operators-catalog deployment has 1 unavailable replicas, redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bde990027a86d67f6f571e540c7c41a4153474968e2d1cf8aaae5f61d4176490","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [certified-operators-catalog, community-operators-catalog, redhat-operators-catalog, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"b81d63019ddcc2109ab747d2bafdb72553b517533208d3b1e2f4932392bed0f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"alert","signature_id":"5e3d7814d9c9dfb8b95ab505c063df649c87d6d6942b6122da0574d5b0d3386c","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T05:12:47Z\n Ended: 2026-09-28T05:57:45Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j4374016-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4374016-mgmt-1\nNamespace: ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"bead9efb4f4f90d639549ff69ed44cddd9403a218b1bccf11f07f0aabe3d53be","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"alert","signature_id":"9a3ac6241850f68acc835ada6fb9a30a428640bf55e1681d17c18117e623ace2","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T05:09:45Z\n Ended: 2026-09-28T05:32:47Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j4374016-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j4374016-mgmt-1\nNamespace: ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"c01da3db1ae269f314546d7e25a338e823ed8481be1395b5a433fb8c541cb106","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"alert","signature_id":"bb574e36c9b0b8ab6f482dba45ea40cb2e48e37b1dc20817e4c37c42065ee2f6","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 22 time(s) (18 unknown, 4 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T05:06:51Z\n Ended: 2026-09-28T05:32:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster\", pod=\"router-7f5bb6dd68-mjsm7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster/router-7f5bb6dd68-mjsm7 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T05:06:53Z\n Ended: 2026-09-28T05:32:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster\", pod=\"router-7f5bb6dd68-9v56k\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3qakt150613r65up3s1cocvavjoksi-agg-cluster/router-7f5bb6dd68-9v56k has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-28T05:06:53Z\n Ended: 2026-09-28T05:31:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9r4v4h8ssovlhfggk1nbpf0t4f5t-b2s4u1s0i0q3z0q\", pod=\"router-79b6987566-srrmw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9r4v4h8ssovlhfggk1nbpf0t4f5t-b2s4u1s0i0q3z0q/router-79b6987566-srrmw has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-28T05:07:56Z\n Ended: 2026-09-28T05:30:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9sbeie1jf1oga9bmqauhj98o2lm6-z7q3j8s4e3j3u3b\", pod=\"router-d8ccb4d9c-4bmnv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9sbeie1jf1oga9bmqauhj98o2lm6-z7q3j8s4e3j3u3b/router-d8ccb4d9c-4bmnv has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-28T05:07:56Z\n Ended: 2026-09-28T05:30:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9sbeie1jf1oga9bmqauhj98o2lm6-z7q3j8s4e3j3u3b\", pod=\"router-d8ccb4d9c-7kk78\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9sbeie1jf1oga9bmqauhj98o2lm6-z7q3j8s4e3j3u3b/router-d8ccb4d9c-7kk78 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-28T05:07:56Z\n Ended: 2026-09-28T05:32:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster\", pod=\"router-78f9bfbb8f-rkcnj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster/router-78f9bfbb8f-rkcnj has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-28T05:07:56Z\n Ended: 2026-09-28T05:30:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9oqqnica740m7cbrv4g0ui9okfrv-v20261001\", pod=\"router-6fc8bdc65f-ldlt9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9oqqnica740m7cbrv4g0ui9okfrv-v20261001/router-6fc8bdc65f-ldlt9 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-28T05:07:56Z\n Ended: 2026-09-28T05:31:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y\", pod=\"router-7f659c45d8-5255k\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y/router-7f659c45d8-5255k has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-28T05:07:56Z\n Ended: 2026-09-28T05:30:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9oqqnica740m7cbrv4g0ui9okfrv-v20261001\", pod=\"router-6fc8bdc65f-zhdpc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9oqqnica740m7cbrv4g0ui9okfrv-v20261001/router-6fc8bdc65f-zhdpc has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-28T05:07:56Z\n Ended: 2026-09-28T05:32:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster\", pod=\"router-78f9bfbb8f-h658c\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster/router-78f9bfbb8f-h658c has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-28T05:07:56Z\n Ended: 2026-09-28T05:31:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y\", pod=\"router-7f659c45d8-d5t8j\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y/router-7f659c45d8-d5t8j has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-28T05:07:56Z\n Ended: 2026-09-28T05:32:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster\", pod=\"router-78f9bfbb8f-txt79\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3qaci6g1hrolig0jqqqo4o0ii7hd2r-cilium-cluster/router-78f9bfbb8f-txt79 has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-28T05:09:53Z\n Ended: 2026-09-28T05:16:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9mk4rs0jgf269qvh2civ1ghcvp1q-q7r1o0f1f9v9t9w\", pod=\"azure-disk-csi-driver-controller-5cdd6fb8fd-qb979\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9mk4rs0jgf269qvh2civ1ghcvp1q-q7r1o0f1f9v9t9w/azure-disk-csi-driver-controller-5cdd6fb8fd-qb979 has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-28T05:12:52Z\n Ended: 2026-09-28T05:21:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9utda6pi90u9tf0cjvsofdad0ucs-i1i3q3s8s2n4k3q\", pod=\"ignition-server-proxy-6b5d65465c-9c7v8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9utda6pi90u9tf0cjvsofdad0ucs-i1i3q3s8s2n4k3q/ignition-server-proxy-6b5d65465c-9c7v8 has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-28T05:12:52Z\n Ended: 2026-09-28T05:18:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9md15na81g26g32hk4dt01vu985b-private-kas\", pod=\"ignition-server-proxy-5c4dd747d9-vm4gw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9md15na81g26g32hk4dt01vu985b-private-kas/ignition-server-proxy-5c4dd747d9-vm4gw has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-28T05:12:52Z\n Ended: 2026-09-28T05:18:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9md15na81g26g32hk4dt01vu985b-private-kas\", pod=\"kube-controller-manager-6b788bd4b6-hnwq2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9md15na81g26g32hk4dt01vu985b-private-kas/kube-controller-manager-6b788bd4b6-hnwq2 has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-28T05:12:52Z\n Ended: 2026-09-28T05:18:53Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9utda6pi90u9tf0cjvsofdad0ucs-i1i3q3s8s2n4k3q\", pod=\"kube-controller-manager-658dbd6bb8-prjzm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9utda6pi90u9tf0cjvsofdad0ucs-i1i3q3s8s2n4k3q/kube-controller-manager-658dbd6bb8-prjzm has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-28T05:30:55Z\n Ended: 2026-09-28T05:36:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t3q9q7agfjgjsadrgs66grkjrrebpou-ephemeral-disk\", pod=\"openshift-apiserver-788cc5f496-6jkkq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2t3q9q7agfjgjsadrgs66grkjrrebpou-ephemeral-disk/openshift-apiserver-788cc5f496-6jkkq has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"c847ab25cc863dfe02ed0da391699a5308a54f8da669d251c09f7b3087a74107","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"a9535031eecea83204e2a98f841c802977ff506d08a46a66165d236a2a58cbfc","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0f2c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-p82fzl2z5vk5\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f41a1e-bd33-4df1-9f47-56824bb74022\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f41a1e-bd33-4df1-9f47-56824bb74022\",\n \"name\": \"03f41a1e-bd33-4df1-9f47-56824bb74022\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:55:14.362679822Z\",\n \"endTime\": \"2026-09-28T05:24:15.510395217Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:107]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f0f2c0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-p82fzl2z5vk5\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f41a1e-bd33-4df1-9f47-56824bb74022\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/03f41a1e-bd33-4df1-9f47-56824bb74022\",\n \"name\": \"03f41a1e-bd33-4df1-9f47-56824bb74022\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:55:14.362679822Z\",\n \"endTime\": \"2026-09-28T05:24:15.510395217Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 2 unavailable replicas, kube-apiserver deployment has 3 unavailable replicas, router deployment has 3 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f63c45d9c3e4d65908b630f98974bc45ae47fd02522293aa9b16d6f8e854d56b","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: WaitingForAvailable: Waiting for Kube APIServer deployment to become available; hosted cluster degraded: UnavailableReplicas: [capi-provider, kube-apiserver, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d4c081228d0f481c6f73f33ac6e9936b572d882bcbe12c0dbf55d138d30bbb5e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"af9b004a3ce14cb7f4279f4399fb02c6d5dcd5d659994a984db66fa7d804f482","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-gng5crkc9wh8/cluster-candidate-4-22-w8wr72 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00117a200\u003e: \n failed to create HCP cluster cluster-candidate-4-22-w8wr72: failed waiting for cluster=\"cluster-candidate-4-22-w8wr72\" in resourcegroup=\"rg-candidate-4-22-gng5crkc9wh8\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00770e12-5d2a-48dc-92a8-bd5616100b1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00770e12-5d2a-48dc-92a8-bd5616100b1d\",\n \"name\": \"00770e12-5d2a-48dc-92a8-bd5616100b1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:03.970731237Z\",\n \"endTime\": \"2026-09-28T05:23:04.188577598Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, router; hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-22-gng5crkc9wh8/cluster-candidate-4-22-w8wr72 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00117a200\u003e: \n failed to create HCP cluster cluster-candidate-4-22-w8wr72: failed waiting for cluster=\"cluster-candidate-4-22-w8wr72\" in resourcegroup=\"rg-candidate-4-22-gng5crkc9wh8\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00770e12-5d2a-48dc-92a8-bd5616100b1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/00770e12-5d2a-48dc-92a8-bd5616100b1d\",\n \"name\": \"00770e12-5d2a-48dc-92a8-bd5616100b1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:03.970731237Z\",\n \"endTime\": \"2026-09-28T05:23:04.188577598Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: redhat-operators-catalog, router; hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog deployment has 1 unavailable replicas, router deployment has 2 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8f36771e9fe45b5eb7816e2eac12b45993b1ce25d5e99408f0060a841a7ead5d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [redhat-operators-catalog, router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"e406fcf9044a8613246d67d24e6056bffd54fdd41a10e4de5e202d3208674c08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"5c839c8ba3fa92aeabac7465b91fbcdcc11b953f6140a77174e182a2f4dccc18","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001904960\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vhq6gmmfrw9m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vhq6gmmfrw9m, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc001904960\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-vhq6gmmfrw9m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-vhq6gmmfrw9m, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-sl29bx8fv8nz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-sl29bx8fv8nz, error: context deadline exceeded"}},{"row_id":"e7ad13c1c574f3859364c87754bb8693a371f4d48118142f8fdd517b1b13cebc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"alert","signature_id":"d3a41ea55a2af8f7aa70bc89be3b1918a824c61a8f8ea40cec168f1ee7e1f5a8","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T05:30:55Z\n Ended: 2026-09-28T05:36:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/services-j4374016\", namespace=\"klusterlet-2t3qakt150613r65up3s1cocvavjoksi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t3qakt150613r65up3s1cocvavjoksi/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-28T05:30:55Z\n Ended: 2026-09-28T05:36:55Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4374016-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.94:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4374016/providers/microsoft.monitor/accounts/services-j4374016\", namespace=\"klusterlet-2t3qakt150613r65up3s1cocvavjoksi\", pod=\"arohcp-monitor-kube-state-metrics-7988dc648c-zzx2c\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2t3qakt150613r65up3s1cocvavjoksi/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"f9b6e88d378c39df6e79bd95447865e3d0fc0d44fb40b5fd8ef85af7b547cd89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"alert","signature_id":"0a3cebc47b4d478e0e1704ea1b9313c7385a4dc3461b6d705f457131c1d86ad3","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T05:08:50Z\n Ended: 2026-09-28T05:58:45Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4374016-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3q9oqqnica740m7cbrv4g0ui9okfrv-v20261001\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4374016-mgmt-2\nNamespace: ocm-arohcpci01-2t3q9oqqnica740m7cbrv4g0ui9okfrv-v20261001\n\nFiring 2:\n State: Resolved\n Started: 2026-09-28T05:14:45Z\n Ended: 2026-09-28T05:44:43Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4374016-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3q9ljvg6g8v7atf8e4uj1uvv1o29vp-z3b4r7r7o7d8c5l\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4374016-mgmt-2\nNamespace: ocm-arohcpci01-2t3q9ljvg6g8v7atf8e4uj1uvv1o29vp-z3b4r7r7o7d8c5l\n\nFiring 3:\n State: Resolved\n Started: 2026-09-28T05:15:43Z\n Ended: 2026-09-28T05:59:48Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4374016-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4374016-mgmt-1\nNamespace: ocm-arohcpci01-2t3q9r41nmns94dh4sm3puhs32psv67j-m1v4i4a9a2m8b6y\n\nFiring 4:\n State: Resolved\n Started: 2026-09-28T05:20:44Z\n Ended: 2026-09-28T05:53:46Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j4374016-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3qa7kalf0nmi953o4ju9ljktm9ed87-i1j4h0b1t2z2z3r\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j4374016-mgmt-2\nNamespace: ocm-arohcpci01-2t3qa7kalf0nmi953o4ju9ljktm9ed87-i1j4h0b1t2z2z3r","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"fa2ea2fefa404fb210bf523ee847d2dd80de8c9e06f97887ad5acaa7ba561b38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104421180684374016","occurred_at":"2026-09-28T04:00:44Z","failed_at":"e2e","signature_id":"9c95fddd44e42f0842cd3dae63f79a607777a45781192a59b8e078313504b0a8","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001698220\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-hqr7q7xq4llr\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e46d4d39-f739-4c7b-9f6e-359435d906f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e46d4d39-f739-4c7b-9f6e-359435d906f6\",\n \"name\": \"e46d4d39-f739-4c7b-9f6e-359435d906f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:08.942854859Z\",\n \"endTime\": \"2026-09-28T05:23:14.16072165Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:81]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001698220\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-hqr7q7xq4llr\" to finish creating: GET https://rp.j4374016.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e46d4d39-f739-4c7b-9f6e-359435d906f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e46d4d39-f739-4c7b-9f6e-359435d906f6\",\n \"name\": \"e46d4d39-f739-4c7b-9f6e-359435d906f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T04:54:08.942854859Z\",\n \"endTime\": \"2026-09-28T05:23:14.16072165Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa7e67c7a590a1275d29fc48bfa890bb54cd4ebb44d206ef3a2ff048e5a51837","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":23,"failure_pattern_ids":["08ad18e89c33b55d4b3e771dd8d81006f379c5497ad8187cd5d751dd18c0ebc4","0dd45aef10b8391c9e5053879220fb6ef98b320a9e44f86e49414d35d0aff7ee","13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","257b448a58141006c11f51362759eea79da6b0398afa6beef61e4326de15f4d6","3b936310d5449729e7fddac902aac9547b4c9f95ec0132ddb43eddb70da02556","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","847f58c2e694684da512b86721767988d4eb7fbb0f488bd687e65af5d9a8f912","8f36771e9fe45b5eb7816e2eac12b45993b1ce25d5e99408f0060a841a7ead5d","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","aa7e67c7a590a1275d29fc48bfa890bb54cd4ebb44d206ef3a2ff048e5a51837","b586f8fea86ba211104459fd89bb5ef2ea79917b36b639db22bd6e7bf973221c","bde990027a86d67f6f571e540c7c41a4153474968e2d1cf8aaae5f61d4176490","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","f63c45d9c3e4d65908b630f98974bc45ae47fd02522293aa9b16d6f8e854d56b"],"matched_occurrences":23,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104397716221071360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7212,"pr_state":"open","pr_sha":"453e2a22f9099079eb9a8ff381022af1c75c569f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T02:28:07Z","started_at":"2026-09-28T02:28:07Z","completed_at":"2026-09-28T04:52:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T05:13:28.341604045Z","timing_metadata_checked_at":"2026-09-28T05:13:28.341604045Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T05:13:28.344631935Z","region_metadata_checked_at":"2026-09-28T05:13:28.344631935Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"84e8f9851b877a693033b369de83dc5c56843fa5d5c00618f53f7fa7868dbf36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104397716221071360","occurred_at":"2026-09-28T02:28:07Z","failed_at":"alert","signature_id":"56897655307efa65f9f11fbfd619611b1b06b0460bb71bb172b6d5e7ce8f3d08","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T03:43:51Z\n Ended: 2026-09-28T03:49:48Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j1071360-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3p00agfmo4aa4d1u3cphgepq43dhn2-v20261001\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j1071360-mgmt-2\nNamespace: ocm-arohcpci01-2t3p00agfmo4aa4d1u3cphgepq43dhn2-v20261001","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"eba6b33509faca34dc5d2792329f52b5dbf98b7ffa731a76e75202d386d918b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7212/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104397716221071360","occurred_at":"2026-09-28T02:28:07Z","failed_at":"e2e","signature_id":"99c1e28c6e269b6801651e5653d78953f7900215867405e5ac5b7f8e3eed035f","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0033b6750\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-nh4vk2wgfp5z\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-nh4vk2wgfp5z, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0033b6750\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-nh4vk2wgfp5z\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-nh4vk2wgfp5z, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-542tb2cfkqw8\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-542tb2cfkqw8, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104394948169175040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"open","pr_sha":"4b0e2b1a8a6df43808b81d3428c15798f49f2218","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-28T02:17:08Z","started_at":"2026-09-28T02:17:08Z","completed_at":"2026-09-28T04:14:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T04:31:56.155263565Z","timing_metadata_checked_at":"2026-09-28T04:31:56.155263565Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T04:31:56.159251652Z","region_metadata_checked_at":"2026-09-28T04:31:56.159251652Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7193/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104389559159623680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7193,"pr_state":"closed","pr_sha":"c45a472db4e317b0509e3ff2987572e72e1f6031","final_merged_sha":"c45a472db4e317b0509e3ff2987572e72e1f6031","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T01:55:43Z","started_at":"2026-09-28T01:55:43Z","completed_at":"2026-09-28T04:07:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T04:10:46.249302328Z","timing_metadata_checked_at":"2026-09-28T04:10:46.249302328Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T04:10:46.25201152Z","region_metadata_checked_at":"2026-09-28T04:10:46.25201152Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104381379776090112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7185,"pr_state":"closed","pr_sha":"84dce2d6dba2b1e1b30987861051d79e7081c5aa","final_merged_sha":"82a0c5b832cb3e6259e0c6c1090e6105a0f71753","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T01:23:13Z","started_at":"2026-09-28T01:23:13Z","completed_at":"2026-09-28T03:35:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T03:39:35.044028963Z","timing_metadata_checked_at":"2026-09-28T03:39:35.044028963Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T03:39:35.047565952Z","region_metadata_checked_at":"2026-09-28T03:39:35.047565952Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"8eed20d6c643a0e842744fa6e28acb1c4ad5ccb82b7b4e5df5bd45d70f6db228","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7185/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104381379776090112","occurred_at":"2026-09-28T01:23:13Z","failed_at":"e2e","signature_id":"817dbbd2f4e987cf783899d57ba9ab565857a252a91d7741c25fbfc506aff5e8","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.003s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104380048239759360","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7178,"pr_state":"open","pr_sha":"f97d6d9b72ab5fe831fe56d01859a4e44a52c87f","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T01:17:54Z","started_at":"2026-09-28T01:17:54Z","completed_at":"2026-09-28T03:34:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T03:39:34.86136974Z","timing_metadata_checked_at":"2026-09-28T03:39:34.86136974Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T03:39:34.866196725Z","region_metadata_checked_at":"2026-09-28T03:39:34.866196725Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"324bcc268f182d8f64869ee4d35e2e99c1255880ccd76fbcea0947cb3114ccdb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104380048239759360","occurred_at":"2026-09-28T01:17:54Z","failed_at":"e2e","signature_id":"968b6078567a646969495cf371728a4af6f1ffed8249b5773e68f74a12927a29","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012c4150\u003e: \n resource group \"complex-cilium-kv-glbv5phs24ph\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-glbv5phs24ph\" to finish deleting: GET https://rp.j9759360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75cb0f96-c479-4ace-abc8-ee149e772174\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75cb0f96-c479-4ace-abc8-ee149e772174\",\n \"name\": \"75cb0f96-c479-4ace-abc8-ee149e772174\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T02:31:25.384468319Z\",\n \"endTime\": \"2026-09-28T02:55:30.931830528Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3nqnt80g6r52fu96h7a5rhdsivrefe still exists (deletion dispatched at 2026-09-28T02:31:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012c4150\u003e: \n resource group \"complex-cilium-kv-glbv5phs24ph\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-glbv5phs24ph\" to finish deleting: GET https://rp.j9759360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75cb0f96-c479-4ace-abc8-ee149e772174\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/75cb0f96-c479-4ace-abc8-ee149e772174\",\n \"name\": \"75cb0f96-c479-4ace-abc8-ee149e772174\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T02:31:25.384468319Z\",\n \"endTime\": \"2026-09-28T02:55:30.931830528Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3nqnt80g6r52fu96h7a5rhdsivrefe still exists (deletion dispatched at 2026-09-28T02:31:27Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ba63473a8759d4818e89dc3253df646ea9627658749a761300942d1d0a165b9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104380048239759360","occurred_at":"2026-09-28T01:17:54Z","failed_at":"e2e","signature_id":"05319646bad3ee1f498711d0d9a69ef24065fb55d47f09c96176ab2ba5be4374","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005607a0\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-555cz2\" in resourcegroup=\"rg-np-downgrade-555cz2-rdlncxrfkhtc\" to finish creating: GET https://rp.j9759360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fc685bb9-f2bf-4194-a9fc-20acce3ce179\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fc685bb9-f2bf-4194-a9fc-20acce3ce179\",\n \"name\": \"fc685bb9-f2bf-4194-a9fc-20acce3ce179\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T02:35:28.867850368Z\",\n \"endTime\": \"2026-09-28T02:54:30.899062424Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w3a0i0z2r1p2g0g-npdg-4-21-pslgq-rs7b4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-555cz2-rdlncxrfkhtc-np-dg-555cz2--managed/providers/Microsoft.Compute/virtualMachines/w3a0i0z2r1p2g0g-npdg-4-21-pslgq-rs7b4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w3a0i0z2r1p2g0g-npdg-4-21-pslgq-rs7b4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-555cz2-rdlncxrfkhtc-np-dg-555cz2--managed/providers/Microsoft.Compute/virtualMachines/w3a0i0z2r1p2g0g-npdg-4-21-pslgq-rs7b4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:600]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005607a0\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-555cz2\" in resourcegroup=\"rg-np-downgrade-555cz2-rdlncxrfkhtc\" to finish creating: GET https://rp.j9759360.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fc685bb9-f2bf-4194-a9fc-20acce3ce179\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fc685bb9-f2bf-4194-a9fc-20acce3ce179\",\n \"name\": \"fc685bb9-f2bf-4194-a9fc-20acce3ce179\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T02:35:28.867850368Z\",\n \"endTime\": \"2026-09-28T02:54:30.899062424Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w3a0i0z2r1p2g0g-npdg-4-21-pslgq-rs7b4: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-555cz2-rdlncxrfkhtc-np-dg-555cz2--managed/providers/Microsoft.Compute/virtualMachines/w3a0i0z2r1p2g0g-npdg-4-21-pslgq-rs7b4 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w3a0i0z2r1p2g0g-npdg-4-21-pslgq-rs7b4: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-downgrade-555cz2-rdlncxrfkhtc-np-dg-555cz2--managed/providers/Microsoft.Compute/virtualMachines/w3a0i0z2r1p2g0g-npdg-4-21-pslgq-rs7b4 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c4aad590e26c04c4f3dd697676f5fb31659d83288090d9541f1bd75191381e5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104380048239759360","occurred_at":"2026-09-28T01:17:54Z","failed_at":"alert","signature_id":"c3b2564dee31ddd5931c6c40cdfa6bad3a8068969e02001e0d6ca7548f201b91","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T02:39:49Z\n Ended: 2026-09-28T02:45:46Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9759360-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9759360-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"e21931608464e38bb667f99bc1d32195565752b5e726bebcda22950f9891eaef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7178/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104380048239759360","occurred_at":"2026-09-28T01:17:54Z","failed_at":"e2e","signature_id":"e3628da3891e1ec49619dde7547596c92e9e1c1cfe73fca8833c1206090506a8","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.002s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104378704682553344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7200,"pr_state":"open","pr_sha":"5286b683d703bf73dc6894532e3d20c8a823679c","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T01:12:35Z","started_at":"2026-09-28T01:12:35Z","completed_at":"2026-09-28T03:27:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T03:39:35.174036052Z","timing_metadata_checked_at":"2026-09-28T03:39:35.174036052Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T03:39:35.176593644Z","region_metadata_checked_at":"2026-09-28T03:39:35.176593644Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b46bf8288308ef0fc6c44d06cd02821c0bc3c11ed45414c87d285ac1e864a0fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7200/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104378704682553344","occurred_at":"2026-09-28T01:12:35Z","failed_at":"e2e","signature_id":"5ffe709affd5c890fbd83d8b5aad5337b8d1b174da24da2cf78360ba40199541","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001585e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-zknh6gjcfdvf\" to finish creating: GET https://rp.j2553344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c774cfb-cf99-4fb2-a28d-fbbb15de258f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c774cfb-cf99-4fb2-a28d-fbbb15de258f\",\n \"name\": \"3c774cfb-cf99-4fb2-a28d-fbbb15de258f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T02:15:44.28328608Z\",\n \"endTime\": \"2026-09-28T02:34:53.308480382Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9y5r9m7v3u4f0e-np-1-hw27d-4ns2q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zknh6gjcfdvf--managed/providers/Microsoft.Compute/virtualMachines/u9y5r9m7v3u4f0e-np-1-hw27d-4ns2q to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9y5r9m7v3u4f0e-np-1-hw27d-4ns2q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zknh6gjcfdvf--managed/providers/Microsoft.Compute/virtualMachines/u9y5r9m7v3u4f0e-np-1-hw27d-4ns2q to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001585e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-zknh6gjcfdvf\" to finish creating: GET https://rp.j2553344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c774cfb-cf99-4fb2-a28d-fbbb15de258f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3c774cfb-cf99-4fb2-a28d-fbbb15de258f\",\n \"name\": \"3c774cfb-cf99-4fb2-a28d-fbbb15de258f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T02:15:44.28328608Z\",\n \"endTime\": \"2026-09-28T02:34:53.308480382Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine u9y5r9m7v3u4f0e-np-1-hw27d-4ns2q: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zknh6gjcfdvf--managed/providers/Microsoft.Compute/virtualMachines/u9y5r9m7v3u4f0e-np-1-hw27d-4ns2q to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine u9y5r9m7v3u4f0e-np-1-hw27d-4ns2q: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-keyvault-zknh6gjcfdvf--managed/providers/Microsoft.Compute/virtualMachines/u9y5r9m7v3u4f0e-np-1-hw27d-4ns2q to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364976289353728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7068,"pr_state":"closed","pr_sha":"5d483559deea65ed903f146fc4f20cbbc522559d","final_merged_sha":"269e5d56bcfac6dc9f8d3ca87b7fb1ea84bf53b2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T00:18:02Z","started_at":"2026-09-28T00:18:02Z","completed_at":"2026-09-28T02:34:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T02:47:41.596291516Z","timing_metadata_checked_at":"2026-09-28T02:47:41.596291516Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T02:47:41.599732205Z","region_metadata_checked_at":"2026-09-28T02:47:41.599732205Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1c04b7456b227b1593830e53254359dc00d4599f1c1091f53071e66a712c6611","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364976289353728","occurred_at":"2026-09-28T00:18:02Z","failed_at":"alert","signature_id":"303fa1869053f8ae8a6eb8a08cc98dd837abeafeb2f0bb2dc5dfcfbf4756a32a","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:46:36Z\n Ended: 2026-09-28T01:54:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9353728-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9353728-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"87077a3ac040e3126c34308f6ce3e3b0b6760f6641e42b15a12794e3978bb75e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364976289353728","occurred_at":"2026-09-28T00:18:02Z","failed_at":"alert","signature_id":"f85bd000c6ca2d2c6689999be19d81766a8baa6d766430077973ac505e4161a1","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:57:15Z\n Ended: 2026-09-28T02:20:15Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j9353728-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3n2c15frlnm47g6kiq5c2e1cd6hcrh-agg-cluster\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j9353728-mgmt-2\nNamespace: ocm-arohcpci01-2t3n2c15frlnm47g6kiq5c2e1cd6hcrh-agg-cluster","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}},{"row_id":"88f203599ee9b2d13b003b060cb0e1a925da83dd67d35143fdf94a81b9d70261","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364976289353728","occurred_at":"2026-09-28T00:18:02Z","failed_at":"e2e","signature_id":"1849503c0698075db16bbbbedb636abe0bb4ab72f8a7bba7219ce1be3d1d71df","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00085bca0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8vl8f9l9j29x\" to finish creating: GET https://rp.j9353728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e5850d6-dc29-4e45-835c-6b9898d7a6dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e5850d6-dc29-4e45-835c-6b9898d7a6dc\",\n \"name\": \"3e5850d6-dc29-4e45-835c-6b9898d7a6dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:19:59.863031878Z\",\n \"endTime\": \"2026-09-28T01:39:01.927768441Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d3p4u4b1l4h8m6c-np-1-bkt5k-rknzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8vl8f9l9j29x--managed/providers/Microsoft.Compute/virtualMachines/d3p4u4b1l4h8m6c-np-1-bkt5k-rknzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d3p4u4b1l4h8m6c-np-1-bkt5k-rknzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8vl8f9l9j29x--managed/providers/Microsoft.Compute/virtualMachines/d3p4u4b1l4h8m6c-np-1-bkt5k-rknzl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00085bca0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-8vl8f9l9j29x\" to finish creating: GET https://rp.j9353728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e5850d6-dc29-4e45-835c-6b9898d7a6dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3e5850d6-dc29-4e45-835c-6b9898d7a6dc\",\n \"name\": \"3e5850d6-dc29-4e45-835c-6b9898d7a6dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:19:59.863031878Z\",\n \"endTime\": \"2026-09-28T01:39:01.927768441Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine d3p4u4b1l4h8m6c-np-1-bkt5k-rknzl: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8vl8f9l9j29x--managed/providers/Microsoft.Compute/virtualMachines/d3p4u4b1l4h8m6c-np-1-bkt5k-rknzl to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine d3p4u4b1l4h8m6c-np-1-bkt5k-rknzl: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/fips-enabled-8vl8f9l9j29x--managed/providers/Microsoft.Compute/virtualMachines/d3p4u4b1l4h8m6c-np-1-bkt5k-rknzl to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364960745263104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6821,"pr_state":"closed","pr_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","final_merged_sha":"115a34116261574c3787f8e7ddc86be68b21cde7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T00:17:58Z","started_at":"2026-09-28T00:17:58Z","completed_at":"2026-09-28T02:26:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T02:36:41.442293889Z","timing_metadata_checked_at":"2026-09-28T02:36:41.442293889Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T02:36:41.44525898Z","region_metadata_checked_at":"2026-09-28T02:36:41.44525898Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"efb88c88ce4f48b99994c413c33eb2fc152ab44869ee239183278d4ad1331a27","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364960745263104","occurred_at":"2026-09-28T00:17:58Z","failed_at":"alert","signature_id":"08b361b6ce0293b567f62579f1da3a35e3254e18d2a7eb29bb5f9ec42f4e8883","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:27:15Z\n Ended: 2026-09-28T01:35:14Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j5263104-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j5263104-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364935222923264","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"closed","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"c7f871c206e6784665943f544e29f3d7e87baeaa","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T00:17:52Z","started_at":"2026-09-28T00:17:52Z","completed_at":"2026-09-28T00:36:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T00:51:28.297444773Z","timing_metadata_checked_at":"2026-09-28T00:51:28.297444773Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-28T00:51:28.300792563Z","region_metadata_checked_at":"2026-09-28T01:54:43.986311183Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c3ab6f3c51d75e8aa5042c8b83624da25d5d107c63ff67af04e9bed28193dbfc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364935222923264","occurred_at":"2026-09-28T00:17:52Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364925383086080","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7157,"pr_state":"closed","pr_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","final_merged_sha":"7dc3b7484131223b5fd5f5dce66a289f85d57f2c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T00:17:50Z","started_at":"2026-09-28T00:17:50Z","completed_at":"2026-09-28T02:26:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T02:36:41.619677326Z","timing_metadata_checked_at":"2026-09-28T02:36:41.619677326Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T02:36:41.622678817Z","region_metadata_checked_at":"2026-09-28T02:36:41.622678817Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"399b8dae19bfadfe91404e81b1984bee5a05cf03b536458c5e81400ab1140457","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364925383086080","occurred_at":"2026-09-28T00:17:50Z","failed_at":"alert","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:14:56Z\n Ended: 2026-09-28T01:33:00Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3086080-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}},{"row_id":"3b2cf3247820da5b0d14f0238369ae63602b03e74abd838be1508aa59e352cb4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364925383086080","occurred_at":"2026-09-28T00:17:50Z","failed_at":"e2e","signature_id":"f20dd3825c25c084156a068aedceedf99a380bc52849b4182a4b8ef151834800","test_name":"Customer should be able to update node pool labels and taints","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002728c00\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-542tb2cfkqw8\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-542tb2cfkqw8, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:186]: failed to update node pool np-1 with new taints and scale up\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002728c00\u003e: \n failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-542tb2cfkqw8\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-542tb2cfkqw8, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-542tb2cfkqw8\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-542tb2cfkqw8, error: context deadline exceeded"}},{"row_id":"68ca16d89babd12c4bc27dde0572a7753610db30f11606fabbb397342cb9412d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364925383086080","occurred_at":"2026-09-28T00:17:50Z","failed_at":"e2e","signature_id":"78fea1fb57019e1048fa9060e13e93e1530c79bf2bc64b62209db632f0720737","test_name":"Customer should be able to update nodepool replicas and autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc003167da0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-kg9lzmtgpr42\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-kg9lzmtgpr42, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:209]: failed to enable autoscaling on node pool np-one-node\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc003167da0\u003e: \n failed waiting for nodepool=\"np-one-node\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-kg9lzmtgpr42\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-one-node in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-kg9lzmtgpr42, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-1\" in cluster=\"cluster-np-labels-taints\" resourcegroup=\"rg-np-labels-taints-542tb2cfkqw8\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-1 in cluster cluster-np-labels-taints in resource group rg-np-labels-taints-542tb2cfkqw8, error: context deadline exceeded"}},{"row_id":"75ee9adc237a46cfbba0e5cef155efda2e2706d127bb5bc67a4d42aead5aab47","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364925383086080","occurred_at":"2026-09-28T00:17:50Z","failed_at":"alert","signature_id":"846cd81d2bf49e1ba6754b4c62a52bdae73d630e941ce6555ae43446efc0f737","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:29:52Z\n Ended: 2026-09-28T01:40:51Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3086080-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3086080-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7174/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104364896954093568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7174,"pr_state":"closed","pr_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","final_merged_sha":"756dbd0aea05ea7d18d18b94d48e2ccdb7df1c9c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T00:17:43Z","started_at":"2026-09-28T00:17:43Z","completed_at":"2026-09-28T02:28:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T02:36:41.798554059Z","timing_metadata_checked_at":"2026-09-28T02:36:41.798554059Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T02:36:41.80121575Z","region_metadata_checked_at":"2026-09-28T02:36:41.80121575Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104363449923407872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7203,"pr_state":"open","pr_sha":"75d72389678ef3781f99923a095f0f5dfe0094f4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T00:11:57Z","started_at":"2026-09-28T00:11:57Z","completed_at":"2026-09-28T02:12:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T02:15:56.540863226Z","timing_metadata_checked_at":"2026-09-28T02:15:56.540863226Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T02:15:56.545157413Z","region_metadata_checked_at":"2026-09-28T02:15:56.545157413Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"029cf98b55234c409b138a92e37c28e894bba57f23c0d037ed09ac828db1bd91","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104363449923407872","occurred_at":"2026-09-28T00:11:57Z","failed_at":"alert","signature_id":"d98ef7ea866f42d46602defe307f0463377933bd3387492c8cad52c2448dec04","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:02:29Z\n Ended: 2026-09-28T01:17:31Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP991h5m\", cluster=\"ci01-j3407872-mgmt-1\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3mqa3q05v2kb8qpc4fuhqft0q3g7f2-s7y7f0s8w4l4q7o\", region=\"canadacentral\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j3407872-mgmt-1\nNamespace: ocm-arohcpci01-2t3mqa3q05v2kb8qpc4fuhqft0q3g7f2-s7y7f0s8w4l4q7o","failure_pattern_match":{"status":"clustered","failure_pattern_id":"76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP991h5m fired"}},{"row_id":"22586f36bbd1ed7e3d43c31b15ad3e0213ca5cda080997e143b20710295e7e0d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104363449923407872","occurred_at":"2026-09-28T00:11:57Z","failed_at":"e2e","signature_id":"11301c96eb4baf025fdfd7d58c7dd82d6896e42f11c7ef2cc1e150fbfe5313df","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c7460\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-vrnhsnxhvbfn\" to finish creating: GET https://rp.j3407872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eaa0085b-c669-46fa-ae33-9cc3f5616168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eaa0085b-c669-46fa-ae33-9cc3f5616168\",\n \"name\": \"eaa0085b-c669-46fa-ae33-9cc3f5616168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:05:43.045871667Z\",\n \"endTime\": \"2026-09-28T01:24:48.358531583Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e7c4q2e4p3k7u1k-np-1-8882r-8hcd9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vrnhsnxhvbfn--managed/providers/Microsoft.Compute/virtualMachines/e7c4q2e4p3k7u1k-np-1-8882r-8hcd9 to exist\\nMachine e7c4q2e4p3k7u1k-np-1-8882r-r4rnq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vrnhsnxhvbfn--managed/providers/Microsoft.Compute/virtualMachines/e7c4q2e4p3k7u1k-np-1-8882r-r4rnq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e7c4q2e4p3k7u1k-np-1-8882r-8hcd9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vrnhsnxhvbfn--managed/providers/Microsoft.Compute/virtualMachines/e7c4q2e4p3k7u1k-np-1-8882r-8hcd9 to exist\\nMachine e7c4q2e4p3k7u1k-np-1-8882r-r4rnq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vrnhsnxhvbfn--managed/providers/Microsoft.Compute/virtualMachines/e7c4q2e4p3k7u1k-np-1-8882r-r4rnq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0004c7460\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-vrnhsnxhvbfn\" to finish creating: GET https://rp.j3407872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eaa0085b-c669-46fa-ae33-9cc3f5616168\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eaa0085b-c669-46fa-ae33-9cc3f5616168\",\n \"name\": \"eaa0085b-c669-46fa-ae33-9cc3f5616168\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:05:43.045871667Z\",\n \"endTime\": \"2026-09-28T01:24:48.358531583Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine e7c4q2e4p3k7u1k-np-1-8882r-8hcd9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vrnhsnxhvbfn--managed/providers/Microsoft.Compute/virtualMachines/e7c4q2e4p3k7u1k-np-1-8882r-8hcd9 to exist\\nMachine e7c4q2e4p3k7u1k-np-1-8882r-r4rnq: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vrnhsnxhvbfn--managed/providers/Microsoft.Compute/virtualMachines/e7c4q2e4p3k7u1k-np-1-8882r-r4rnq to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine e7c4q2e4p3k7u1k-np-1-8882r-8hcd9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vrnhsnxhvbfn--managed/providers/Microsoft.Compute/virtualMachines/e7c4q2e4p3k7u1k-np-1-8882r-8hcd9 to exist\\nMachine e7c4q2e4p3k7u1k-np-1-8882r-r4rnq: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-vrnhsnxhvbfn--managed/providers/Microsoft.Compute/virtualMachines/e7c4q2e4p3k7u1k-np-1-8882r-r4rnq to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c57419e780068e76978cbdcdd33910a731f08e56ec62ddda66d4f6d6da94e2bd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104363449923407872","occurred_at":"2026-09-28T00:11:57Z","failed_at":"alert","signature_id":"73e11070f2a858d26270686b98d03cc56fbf8cc31e4b226ab179fe754a016a71","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:28:29Z\n Ended: 2026-09-28T01:34:32Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3407872-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3407872-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"c6ecd6b3fdc64d3216e3bcf52adc470ad5b405eed972e271bed65cc30a4b0bed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7203/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104363449923407872","occurred_at":"2026-09-28T00:11:57Z","failed_at":"alert","signature_id":"0746757cae55e1359ac94dc358bff8c1481c3785d7e5878a5a71e4eab63b083e","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdReadLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:05:30Z\n Ended: 2026-09-28T01:42:31Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdReadLatencyP996h30m\", cluster=\"ci01-j3407872-mgmt-1\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3mqa3q05v2kb8qpc4fuhqft0q3g7f2-s7y7f0s8w4l4q7o\", region=\"canadacentral\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-read-latency\"\n Description: etcd gRPC Range (read) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j3407872-mgmt-1\nNamespace: ocm-arohcpci01-2t3mqa3q05v2kb8qpc4fuhqft0q3g7f2-s7y7f0s8w4l4q7o","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","failure_pattern":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdReadLatencyP996h30m fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","76d673befde0db0588f6d89724101698e8d234002b0e3ac8850257675218248e","d9b11fe77d6c0658c4e20b8298b53bea3d0e6340f683ca0f2f3491bd0320501d","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104362505772994560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7202,"pr_state":"open","pr_sha":"bf798191f1e4784ba8c222bbae084b193940b0ff","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T00:08:13Z","started_at":"2026-09-28T00:08:13Z","completed_at":"2026-09-28T02:19:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T02:26:38.267584374Z","timing_metadata_checked_at":"2026-09-28T02:26:38.267584374Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T02:26:38.270285465Z","region_metadata_checked_at":"2026-09-28T02:26:38.270285465Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"35046fd2621c2e11908dff572c187464e3fb94497bd673fdc10f29428c8c80a1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104362505772994560","occurred_at":"2026-09-28T00:08:13Z","failed_at":"alert","signature_id":"0e04fa5714858ca45f181fa5b8f6c4124ef2770063cbac0e9be7b8c170fcd30e","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP991h5m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:06:00Z\n Ended: 2026-09-28T01:26:57Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP991h5m\", cluster=\"ci01-j2994560-mgmt-2\", component=\"etcd\", long_window=\"1h\", namespace=\"ocm-arohcpci01-2t3mnh30341mbni3cq0802m7mt0jj8ii-x2a5l3m8n0n6f6h\", region=\"westus3\", severity=\"3\", short_window=\"5m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 1h and 5m windows.\nManagement Cluster: ci01-j2994560-mgmt-2\nNamespace: ocm-arohcpci01-2t3mnh30341mbni3cq0802m7mt0jj8ii-x2a5l3m8n0n6f6h","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP991h5m fired"}},{"row_id":"9bd3ab465e2846897e6855869d9d64753bd7321f13f9b3b7a76276cccca81a16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104362505772994560","occurred_at":"2026-09-28T00:08:13Z","failed_at":"alert","signature_id":"d1fe2c91f414126eebe9264ae5312e44cb5f74a34c9a0915d3a6a984b56ea6aa","test_name":"[aro-hcp-observability] [hcp] alert userJourneyEtcdWriteLatencyP996h30m does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-28T01:08:59Z\n Ended: 2026-09-28T01:28:10Z\n Severity: Sev3\n Labels: alertname=\"userJourneyEtcdWriteLatencyP996h30m\", cluster=\"ci01-j2994560-mgmt-2\", component=\"etcd\", long_window=\"6h\", namespace=\"ocm-arohcpci01-2t3mnh30341mbni3cq0802m7mt0jj8ii-x2a5l3m8n0n6f6h\", region=\"westus3\", severity=\"3\", short_window=\"30m\", slo=\"etcd-grpc-write-latency\"\n Description: etcd gRPC Txn (write) P99 latency has exceeded 500ms in both the 6h and 30m windows.\nManagement Cluster: ci01-j2994560-mgmt-2\nNamespace: ocm-arohcpci01-2t3mnh30341mbni3cq0802m7mt0jj8ii-x2a5l3m8n0n6f6h","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","failure_pattern":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired","search_query":"alert [hcp] userJourneyEtcdWriteLatencyP996h30m fired"}},{"row_id":"f82db1b5d9d2c163c38a29c58cfbb36c203f9c706a80c0672c9909f4e2087da3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7202/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104362505772994560","occurred_at":"2026-09-28T00:08:13Z","failed_at":"e2e","signature_id":"828834b6dd0e71a9b413e9e910f9937d0125a741c626c11c5bfe7e11833f2e4f","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.005s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["6bd1afd6570e82a62edcedc5b7fa9eb76b6280f40beb98309f35479bc089455c","9ec965b164c5e342982eb3ee42576447c4cea10d74f342ccab7c85d1db27287c","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104360746694479872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7041,"pr_state":"open","pr_sha":"4b0e2b1a8a6df43808b81d3428c15798f49f2218","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-28T00:00:40Z","started_at":"2026-09-28T00:00:40Z","completed_at":"2026-09-28T02:14:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T02:26:38.103398311Z","timing_metadata_checked_at":"2026-09-28T02:26:38.103398311Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T02:26:38.106267401Z","region_metadata_checked_at":"2026-09-28T02:26:38.106267401Z"},"failed_at":["e2e"],"failed_test_count":4,"occurrences":[{"row_id":"471e3e7545588bb184e16b413c2b1ba6cf029e75f41458220f880f2843ac0c85","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104360746694479872","occurred_at":"2026-09-28T00:00:40Z","failed_at":"e2e","signature_id":"b3295d25e69eb7628cbb4ab513ba3757584094fbc4dc76c3f57f93b86af34242","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001806078\u003e: \n resource group \"admin-credential-lifecycle-test-2mncmbxff48k\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-gzn4lc\" in resourcegroup=\"admin-credential-lifecycle-test-2mncmbxff48k\" to finish deleting: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/100c82ed-e364-4103-9093-3cc047de25dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/100c82ed-e364-4103-9093-3cc047de25dc\",\n \"name\": \"100c82ed-e364-4103-9093-3cc047de25dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:00:46.173719966Z\",\n \"endTime\": \"2026-09-28T01:24:48.870398522Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3mm6bjna4825qvibg4tmtpm7c95ke0 still exists (deletion dispatched at 2026-09-28T01:00:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001806078\u003e: \n resource group \"admin-credential-lifecycle-test-2mncmbxff48k\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"admin-cred-lifecycle-gzn4lc\" in resourcegroup=\"admin-credential-lifecycle-test-2mncmbxff48k\" to finish deleting: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/100c82ed-e364-4103-9093-3cc047de25dc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/100c82ed-e364-4103-9093-3cc047de25dc\",\n \"name\": \"100c82ed-e364-4103-9093-3cc047de25dc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:00:46.173719966Z\",\n \"endTime\": \"2026-09-28T01:24:48.870398522Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3mm6bjna4825qvibg4tmtpm7c95ke0 still exists (deletion dispatched at 2026-09-28T01:00:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5bf63a63dd5c0e71ac57d61bf4be19ae3108de01020aa903be9a83cf77ee1eeb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104360746694479872","occurred_at":"2026-09-28T00:00:40Z","failed_at":"e2e","signature_id":"2e6206962622fa23645da22c99b31d5211fa88fc0db515931249a87ce69c3e13","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000ca85b8\u003e: \n resource group \"admin-api-serialconsole-st5xdz2xcc6m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-st5xdz2xcc6m\" to finish deleting: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d9aeeb2-65e9-44c7-90ba-68e9e692244c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d9aeeb2-65e9-44c7-90ba-68e9e692244c\",\n \"name\": \"3d9aeeb2-65e9-44c7-90ba-68e9e692244c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:06:48.076247478Z\",\n \"endTime\": \"2026-09-28T01:30:48.731406356Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3mmeectlf6lq0ip66b1ckuumq6c71t still exists (deletion dispatched at 2026-09-28T01:06:49Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000ca85b8\u003e: \n resource group \"admin-api-serialconsole-st5xdz2xcc6m\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-st5xdz2xcc6m\" to finish deleting: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d9aeeb2-65e9-44c7-90ba-68e9e692244c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/3d9aeeb2-65e9-44c7-90ba-68e9e692244c\",\n \"name\": \"3d9aeeb2-65e9-44c7-90ba-68e9e692244c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:06:48.076247478Z\",\n \"endTime\": \"2026-09-28T01:30:48.731406356Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3mmeectlf6lq0ip66b1ckuumq6c71t still exists (deletion dispatched at 2026-09-28T01:06:49Z); [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"669f8a475e6f3ce9d7dd38c6b8315c81cad58167ec43a621ab411d3132e415c5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104360746694479872","occurred_at":"2026-09-28T00:00:40Z","failed_at":"e2e","signature_id":"ab546a099515b9c90776a9116c547ec830827b9274733c363741cfc037e250fc","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014796a0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qq55pmfqfj46\" to finish creating: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1b5c158-195f-45c4-8138-3df46250e606\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1b5c158-195f-45c4-8138-3df46250e606\",\n \"name\": \"e1b5c158-195f-45c4-8138-3df46250e606\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:10:15.13644965Z\",\n \"endTime\": \"2026-09-28T01:29:18.755471449Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine h2y7r3j2i6e7b9v-autoscale-noaz-kch6d-5xtbd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-qq55pmfqfj46--managed/providers/Microsoft.Compute/virtualMachines/h2y7r3j2i6e7b9v-autoscale-noaz-kch6d-5xtbd to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine h2y7r3j2i6e7b9v-autoscale-noaz-kch6d-5xtbd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-qq55pmfqfj46--managed/providers/Microsoft.Compute/virtualMachines/h2y7r3j2i6e7b9v-autoscale-noaz-kch6d-5xtbd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014796a0\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qq55pmfqfj46\" to finish creating: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1b5c158-195f-45c4-8138-3df46250e606\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e1b5c158-195f-45c4-8138-3df46250e606\",\n \"name\": \"e1b5c158-195f-45c4-8138-3df46250e606\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T01:10:15.13644965Z\",\n \"endTime\": \"2026-09-28T01:29:18.755471449Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine h2y7r3j2i6e7b9v-autoscale-noaz-kch6d-5xtbd: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-qq55pmfqfj46--managed/providers/Microsoft.Compute/virtualMachines/h2y7r3j2i6e7b9v-autoscale-noaz-kch6d-5xtbd to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine h2y7r3j2i6e7b9v-autoscale-noaz-kch6d-5xtbd: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-qq55pmfqfj46--managed/providers/Microsoft.Compute/virtualMachines/h2y7r3j2i6e7b9v-autoscale-noaz-kch6d-5xtbd to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c449524bb50eec56d12ca9e0286bec5a83e84b0b0383c0917d37ec369cd74d4a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7041/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2104360746694479872","occurred_at":"2026-09-28T00:00:40Z","failed_at":"e2e","signature_id":"d03f3c6aca7be3f6a3c7d8785334ad1db8dd496853f706eb72adae638570308d","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001948060\u003e: \n resource group \"customer-rg-hx6m7trvhcw8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hx6m7trvhcw8\" to finish deleting: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/010ae2fe-5ec0-4b79-bfe4-03112aab36d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/010ae2fe-5ec0-4b79-bfe4-03112aab36d3\",\n \"name\": \"010ae2fe-5ec0-4b79-bfe4-03112aab36d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T00:59:43.756752481Z\",\n \"endTime\": \"2026-09-28T01:23:48.800910697Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3mmkbf6c7cpr22sjfg901rr6tj3ald still exists (deletion dispatched at 2026-09-28T00:59:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001948060\u003e: \n resource group \"customer-rg-hx6m7trvhcw8\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"basic-hcp-cluster2\" in resourcegroup=\"customer-rg-hx6m7trvhcw8\" to finish deleting: GET https://rp.j4479872.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/010ae2fe-5ec0-4b79-bfe4-03112aab36d3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/010ae2fe-5ec0-4b79-bfe4-03112aab36d3\",\n \"name\": \"010ae2fe-5ec0-4b79-bfe4-03112aab36d3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T00:59:43.756752481Z\",\n \"endTime\": \"2026-09-28T01:23:48.800910697Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t3mmkbf6c7cpr22sjfg901rr6tj3ald still exists (deletion dispatched at 2026-09-28T00:59:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["32765c1a23e12565fb757f5ef35da7abebb4b281f287f8b70ea50837d646b9c3","63a24fc72ea6be67478ba57ceca4f83bfef60aaeae2826b2abf0c75f70bec403","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}}]},{"environment":"int","summary":{"total_runs":3,"failed_runs":1,"runs_with_occurrences":1,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104703363684765696","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T22:42:40Z","started_at":"2026-09-28T22:42:40Z","completed_at":"2026-09-29T00:58:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:14:28.043094247Z","timing_metadata_checked_at":"2026-09-29T01:14:28.043094247Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:14:28.045979137Z","region_metadata_checked_at":"2026-09-29T01:14:28.045979137Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104667201544916992","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T20:18:58Z","started_at":"2026-09-28T20:18:58Z","completed_at":"2026-09-28T22:35:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T22:46:16.853289106Z","timing_metadata_checked_at":"2026-09-28T22:46:16.853289106Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T22:46:16.856294097Z","region_metadata_checked_at":"2026-09-28T22:46:16.856294097Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"d7d3db6ce5b766049a2ab296274fa1ca22b86b408290c400790b42e62235f9b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104667201544916992","occurred_at":"2026-09-28T20:18:58Z","failed_at":"e2e","signature_id":"1e10870f9142e34bc1040827a466038cf22783d022da97c635df733bee6dca9b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fnv7lb\" for cluster \"cluster-candidate-5-0-fnv7lb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00024f2c0\u003e: \n failed to create NodePool np-fnv7lb: failed waiting for nodepool=\"np-fnv7lb\" for cluster \"cluster-candidate-5-0-fnv7lb\" in resourcegroup=\"rg-candidate-5-0-v6nvmt4rxzjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/203ad465-1e1f-4337-9659-f5fe2972a72a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/203ad465-1e1f-4337-9659-f5fe2972a72a\",\n \"name\": \"203ad465-1e1f-4337-9659-f5fe2972a72a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T21:17:06.138034027Z\",\n \"endTime\": \"2026-09-28T21:36:06.809868093Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-v6nvmt4rxzjw-candidate-fnv7lb--managed/providers/Microsoft.Compute/virtualMachines/w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-v6nvmt4rxzjw-candidate-fnv7lb--managed/providers/Microsoft.Compute/virtualMachines/w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-fnv7lb\" for cluster \"cluster-candidate-5-0-fnv7lb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00024f2c0\u003e: \n failed to create NodePool np-fnv7lb: failed waiting for nodepool=\"np-fnv7lb\" for cluster \"cluster-candidate-5-0-fnv7lb\" in resourcegroup=\"rg-candidate-5-0-v6nvmt4rxzjw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/203ad465-1e1f-4337-9659-f5fe2972a72a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/203ad465-1e1f-4337-9659-f5fe2972a72a\",\n \"name\": \"203ad465-1e1f-4337-9659-f5fe2972a72a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T21:17:06.138034027Z\",\n \"endTime\": \"2026-09-28T21:36:06.809868093Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-v6nvmt4rxzjw-candidate-fnv7lb--managed/providers/Microsoft.Compute/virtualMachines/w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-v6nvmt4rxzjw-candidate-fnv7lb--managed/providers/Microsoft.Compute/virtualMachines/w5k5h9q4v3j3g4r-np-fnv7lb-b5wnn-tkxbb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["91ddd205ea7689b089e73b7d7ce728678548484f191065bcd129e0cd4395deae"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2104614217746223104","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T16:48:25Z","started_at":"2026-09-28T16:48:25Z","completed_at":"2026-09-28T19:16:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T19:25:11.880112779Z","timing_metadata_checked_at":"2026-09-28T19:25:11.880112779Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T19:25:11.883123369Z","region_metadata_checked_at":"2026-09-28T19:25:11.883123369Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":13,"failed_runs":7,"runs_with_occurrences":7,"runs_with_failure_pattern_matches":7,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":6,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T23:22:35Z","started_at":"2026-09-28T23:22:35Z","completed_at":"2026-09-29T01:47:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:55:31.127655818Z","timing_metadata_checked_at":"2026-09-29T01:55:31.127655818Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:55:31.130521909Z","region_metadata_checked_at":"2026-09-29T01:55:31.130521909Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"48b59f3875e9ba2e31c5adcb9ded0486e232ebf2e75ecb6769908260d7c9b23c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","occurred_at":"2026-09-28T23:22:35Z","failed_at":"e2e","signature_id":"29bde2d7b8d5a41edc15f4349385c2d0099553bdf943c64bb8c46da6c996a3e1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00087e120\u003e: \n failed to create NodePool np-rcls6m: failed waiting for nodepool=\"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\" in resourcegroup=\"rg-candidate-4-21-92w77dwp5rg8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n \"name\": \"a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:16:31.628142216Z\",\n \"endTime\": \"2026-09-29T00:35:38.838060326Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00087e120\u003e: \n failed to create NodePool np-rcls6m: failed waiting for nodepool=\"np-rcls6m\" for cluster \"cluster-candidate-4-21-rcls6m\" in resourcegroup=\"rg-candidate-4-21-92w77dwp5rg8\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n \"name\": \"a337697b-a055-47ea-9ec2-d9638cfc3ab2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:16:31.628142216Z\",\n \"endTime\": \"2026-09-29T00:35:38.838060326Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-92w77dwp5rg8-candidate-rcls6m--managed/providers/Microsoft.Compute/virtualMachines/q8k7t5p0o1m8l5r-np-rcls6m-wc8xf-q85q8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dd115327b69c3b100b2ab8f12a8a65af701b609a8befa9d6488273ed31b7c8ca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104713411836252160","occurred_at":"2026-09-28T23:22:35Z","failed_at":"e2e","signature_id":"057d78f20fc782ce9997cb17d11d93f7004a1d7a6ecbd74b197c32efd5c0100d","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b43f40\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b43f40\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T23:11:16Z","started_at":"2026-09-28T23:11:16Z","completed_at":"2026-09-29T01:51:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T02:06:08.08290612Z","timing_metadata_checked_at":"2026-09-29T02:06:08.08290612Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T02:06:08.0887953Z","region_metadata_checked_at":"2026-09-29T02:06:08.0887953Z"},"failed_at":["e2e"],"failed_test_count":5,"occurrences":[{"row_id":"0dbd96b95b7f6de6d927e9f8622c4c6d6178fc538445f547e0865b8456950faa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"9b958b779ca895c4c49a73903b6b72b8677fe49ebfe23b8787341a9e83c6884a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010410e0\u003e: \n failed to create NodePool np-5m4wlb: failed waiting for nodepool=\"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\" in resourcegroup=\"rg-candidate-4-23-wwwsqs8cghpq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\",\n \"name\": \"91d9873c-e44e-4e44-ba41-ade87f500052\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:18:19.018330415Z\",\n \"endTime\": \"2026-09-29T00:37:28.829514453Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: Creating: virtualmachine creating or updating\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010410e0\u003e: \n failed to create NodePool np-5m4wlb: failed waiting for nodepool=\"np-5m4wlb\" for cluster \"cluster-candidate-4-23-5m4wlb\" in resourcegroup=\"rg-candidate-4-23-wwwsqs8cghpq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/91d9873c-e44e-4e44-ba41-ade87f500052\",\n \"name\": \"91d9873c-e44e-4e44-ba41-ade87f500052\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:18:19.018330415Z\",\n \"endTime\": \"2026-09-29T00:37:28.829514453Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-jdc42: Creating: virtualmachine creating or updating\\nMachine h9a1g7g5s5w9p9g-np-5m4wlb-4q46f-tdh4q: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"46ec4dc1ee77f2b04feaf8a4e870acc27d6af0c3513836a206a1d5f95dff9612","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"0950bf1fa4b422939bc22e6eec036cdf20bb4658d9c093acba892685503bea17","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bad000\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-64htmh\" in resourcegroup=\"rg-np-noedge-64htmh-c4j9qdqfrq4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n \"name\": \"3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T01:00:25.564607872Z\",\n \"endTime\": \"2026-09-29T01:19:27.08336887Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: Creating: virtualmachine creating or updating\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:321]: failed to create nodepool npnoedge-4-20 at version 4.20.39\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bad000\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-64htmh\" in resourcegroup=\"rg-np-noedge-64htmh-c4j9qdqfrq4j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n \"name\": \"3b426bd0-3f4b-4ee7-96a0-7d54b3579d89\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T01:00:25.564607872Z\",\n \"endTime\": \"2026-09-29T01:19:27.08336887Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-md5vk: Creating: virtualmachine creating or updating\\nMachine w1j2z1c7q6g2g6k-npnoedge-4-20-7rq4d-mvq9g: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"6681890728c7562f98f797c9fff59b9157e7a6a75b3b89f7fde64576735f17ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"e4912b2f6d7e7563f8767c16448cd57abe8e92db7831d87c4f122e3d8fc2c77d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-p28f7h\" for cluster \"cluster-candidate-4-22-p28f7h\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00068b3c0\u003e: \n failed to create NodePool np-p28f7h: failed waiting for nodepool=\"np-p28f7h\" for cluster \"cluster-candidate-4-22-p28f7h\" in resourcegroup=\"rg-candidate-4-22-mrpj7hdjdh7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/45638889-70a7-4fdc-8d99-520c3c98a19b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/45638889-70a7-4fdc-8d99-520c3c98a19b\",\n \"name\": \"45638889-70a7-4fdc-8d99-520c3c98a19b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:17:24.13464273Z\",\n \"endTime\": \"2026-09-29T00:36:28.832441888Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a9f5u6l9p6p9x0x-np-p28f7h-pwbqm-k6srx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a9f5u6l9p6p9x0x-np-p28f7h-pwbqm-k6srx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-p28f7h\" for cluster \"cluster-candidate-4-22-p28f7h\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00068b3c0\u003e: \n failed to create NodePool np-p28f7h: failed waiting for nodepool=\"np-p28f7h\" for cluster \"cluster-candidate-4-22-p28f7h\" in resourcegroup=\"rg-candidate-4-22-mrpj7hdjdh7l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/45638889-70a7-4fdc-8d99-520c3c98a19b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/45638889-70a7-4fdc-8d99-520c3c98a19b\",\n \"name\": \"45638889-70a7-4fdc-8d99-520c3c98a19b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:17:24.13464273Z\",\n \"endTime\": \"2026-09-29T00:36:28.832441888Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine a9f5u6l9p6p9x0x-np-p28f7h-pwbqm-k6srx: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine a9f5u6l9p6p9x0x-np-p28f7h-pwbqm-k6srx: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"8dc3aee5dc9358deaaabc1cedebb19a6707e4f408d61dc9277f1c417f5cfbae7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"f07ab4b774d34743455c8d173177f39c64d2818384324244b685d13e7ea7d3cc","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0007275a0\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdcdkl9rvn5l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\",\n \"name\": \"3a70e081-8150-479f-812b-31eb02522612\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T01:01:02.974562787Z\",\n \"endTime\": \"2026-09-29T01:20:06.931508754Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0007275a0\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-wdcdkl9rvn5l\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/3a70e081-8150-479f-812b-31eb02522612\",\n \"name\": \"3a70e081-8150-479f-812b-31eb02522612\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T01:01:02.974562787Z\",\n \"endTime\": \"2026-09-29T01:20:06.931508754Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine h3s5a5u1s5o4h9w-np-deletable-9nx5r-bcsqf: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management"}},{"row_id":"a44157a6759cdfba7890fa3a3cdd707a5092f989f44725faa6698883b24cfd89","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104710565229236224","occurred_at":"2026-09-28T23:11:16Z","failed_at":"e2e","signature_id":"053123c7443d1054083dcab559d2546849eae900c250f58b40fbf47904d636dc","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7f080\u003e: \n failed to create NodePool np-bhxv8b: failed waiting for nodepool=\"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\" in resourcegroup=\"rg-candidate-4-21-mrwxnrjtcxks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n \"name\": \"b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:16:53.990233971Z\",\n \"endTime\": \"2026-09-29T00:35:58.987434024Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b7f080\u003e: \n failed to create NodePool np-bhxv8b: failed waiting for nodepool=\"np-bhxv8b\" for cluster \"cluster-candidate-4-21-bhxv8b\" in resourcegroup=\"rg-candidate-4-21-mrwxnrjtcxks\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n \"name\": \"b5cb7fd8-b127-4539-8778-6e7d5e5f9f3d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:16:53.990233971Z\",\n \"endTime\": \"2026-09-29T00:35:58.987434024Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l9g0x5l7a1n1v5a-np-bhxv8b-5zb4x-km8kk: Creating: virtualmachine creating or updating\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","dd31a666427de8559960a5a579ef213ff8cea3cb0284f227588522fa5999c7c9"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T23:00:26Z","started_at":"2026-09-28T23:00:26Z","completed_at":"2026-09-29T00:50:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:04:03.677054674Z","timing_metadata_checked_at":"2026-09-29T01:04:03.677054674Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:04:03.679909965Z","region_metadata_checked_at":"2026-09-29T01:04:03.679909965Z"},"failed_at":["e2e"],"failed_test_count":14,"occurrences":[{"row_id":"1b66d8f3a5c143c96942e5ece4d9e241960b248975191a56cffb98a298ebfc2c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"e7f9b8ac38554f62beb05357bcb84d1c79a8ef404e325310c520ae6e5285e61d","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f35488\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.008s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000f35488\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"25ac2f0b85e97a163367bb2843c94b3527c743e45785d1a5f23675cca738d2e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"7732b8465eb0b310323c46363238564ae3708ad1dd66f83e67d67744e2112be2","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00179cc60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-v4rx8l2tw9wq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00179cc60\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-v4rx8l2tw9wq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"46ff0b3b33bfe312808502d2dd00d17cb9aff1476f477652ded7346401818c6a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"9aafea9b5d579d12248bc117bbaa084f2543cc7e5eb735c49007499bd1c1c528","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000aa43c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qgqdfprzx89s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000aa43c0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-qgqdfprzx89s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"5abe172d06b02fda37f1134af66e71d315c5b76f67bf0b45846b465a48d3f971","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"56f51dbe4172c7698c7e2ac9b44cc9309db978fcf5349649586b91b53538f803","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048440\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-k6t82b\" resourcegroup=\"rg-np-dg-minor-k6t82b-4f2hjngd28k4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-k6t82b-4f2hjngd28k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-k6t82b/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048440\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-k6t82b\" resourcegroup=\"rg-np-dg-minor-k6t82b-4f2hjngd28k4\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-k6t82b-4f2hjngd28k4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-k6t82b/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6219e83d01cce0fe664189ba73a5771ea599b137e585f4fc68fcc60f211a082d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"8bfd58413915673e86f8668f73c4a1593d6b25ac14d3df8cca7f16fa47901f61","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d24340\u003e: \n failed to create NodePool np-5cw4w9: failed starting nodepool creation \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\" in resourcegroup=\"rg-nightly-4-20-b7gtl2svm55n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-b7gtl2svm55n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-5cw4w9/nodePools/np-5cw4w9\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d24340\u003e: \n failed to create NodePool np-5cw4w9: failed starting nodepool creation \"np-5cw4w9\" for cluster \"cluster-nightly-4-20-5cw4w9\" in resourcegroup=\"rg-nightly-4-20-b7gtl2svm55n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-b7gtl2svm55n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-5cw4w9/nodePools/np-5cw4w9\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"6afb64dd831e95d14afa16946a4d93d3e2fa42baf141d0f4ddd715171ad8fb94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"f675dd7539b7a7c84d8073b82c8ff50aab68a3afafa2637412a75cd680bb8823","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0024d21b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gd648czgpsjb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0024d21b0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-gd648czgpsjb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"84ab21d5da097a4526804ad15e8f9cffda43e128c39580ec8184c1e7c3d7e8a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"61da179f0b7befc0f6ef4e2c530e9e4e588275b255eb735499eff0b6124e8ce5","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001055c80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tf5tm29lbrgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n \"name\": \"51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:10:33.615095623Z\",\n \"endTime\": \"2026-09-28T23:29:40.114080242Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001055c80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tf5tm29lbrgl\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n \"name\": \"51bc36e8-ffb9-42f8-8bc5-c2b8cb5c2386\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:10:33.615095623Z\",\n \"endTime\": \"2026-09-28T23:29:40.114080242Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2fhfr-24lbg: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tf5tm29lbrgl--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2fhfr-24lbg to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management"}},{"row_id":"850fea80834522a2f88a1d806416b86cdd1bb884b6af42102c651dc6a93818c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"5e02d079f2bb2b4baf359fdcea15f52be4ed619abb6034bc78cd714b27a3f342","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ade640\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-t7jl46\" in resourcegroup=\"rg-np-version-upgrade-t7jl46-65khsnvbc4zl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t7jl46-65khsnvbc4zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t7jl46/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ade640\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-t7jl46\" in resourcegroup=\"rg-np-version-upgrade-t7jl46-65khsnvbc4zl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-t7jl46-65khsnvbc4zl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-t7jl46/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"dbd368e1e7ed04d70004c589fa81a2bd37ea295dd551b5b55e32d742d8811bb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"8de98f59efa15f72a396ca1bd7e2dff75eb6bfdd7ab655e62dff7e62b3c1152d","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00130ef60\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7g7qln9l2np5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n \"name\": \"ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:42:15.584467675Z\",\n \"endTime\": \"2026-09-29T00:01:22.906593334Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00130ef60\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-7g7qln9l2np5\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n \"name\": \"ebbacb44-7cd1-425c-b348-f154c9c9ba46\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:42:15.584467675Z\",\n \"endTime\": \"2026-09-29T00:01:22.906593334Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ea-cluster-ea-np-1-4fbc8-5n6n7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-7g7qln9l2np5--managed/providers/Microsoft.Compute/virtualMachines/ea-cluster-ea-np-1-4fbc8-5n6n7 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"dc30b5999fa75fa9faa92b3403f8581713a3bb1da1f30dd90649307bc3651c16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"8bea3b31627e5d847815834266e4adb45e346c49dee4938fc8669a5a0d148891","test_name":"Customer should be able to delete a nodepool","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0030a64a0\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kx45mg6gnp52\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\",\n \"name\": \"7aa70862-e406-44df-915d-27100318e68b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:56:08.330966898Z\",\n \"endTime\": \"2026-09-29T00:15:12.920582798Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0030a64a0\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-kx45mg6gnp52\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/7aa70862-e406-44df-915d-27100318e68b\",\n \"name\": \"7aa70862-e406-44df-915d-27100318e68b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:56:08.330966898Z\",\n \"endTime\": \"2026-09-29T00:15:12.920582798Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-kx45mg6gnp52--managed/providers/Microsoft.Compute/virtualMachines/e4w5q3g8d6k8y5e-np-deletable-rkrhk-wfjcc to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management"}},{"row_id":"e320dea94e37550b9886781be4c378f5098a37633e42caa8dc6ecaf70e575e2a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"c36a633bcd07dffb48200bace887b57ae6fe20eba4fb59a0da31d14814ce0b59","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000831500\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-zvgs4q\" in resourcegroup=\"rg-np-skip-minor-zvgs4q-797p7px4gkts\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-zvgs4q-797p7px4gkts/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-zvgs4q/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000831500\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-zvgs4q\" in resourcegroup=\"rg-np-skip-minor-zvgs4q-797p7px4gkts\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-zvgs4q-797p7px4gkts/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-zvgs4q/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f155dc3cecff24d24c50582a0a3b3936d18432ea8ccdbfd78cb0a962cbe22de8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"d33cdf557cf6b0a812334a9c133b09038e1b46fff4e53cc99d59b39fb3a25778","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-q9zwpnrqm4zb/cluster-nightly-5-1-sq64f4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110cda0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-sq64f4: failed waiting for cluster=\"cluster-nightly-5-1-sq64f4\" in resourcegroup=\"rg-nightly-5-1-q9zwpnrqm4zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\",\n \"name\": \"25287b3f-b870-4e18-a51f-514432892aec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:31:45.913312Z\",\n \"endTime\": \"2026-09-29T00:00:52.759469958Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-q9zwpnrqm4zb/cluster-nightly-5-1-sq64f4 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110cda0\u003e: \n failed to create HCP cluster cluster-nightly-5-1-sq64f4: failed waiting for cluster=\"cluster-nightly-5-1-sq64f4\" in resourcegroup=\"rg-nightly-5-1-q9zwpnrqm4zb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/25287b3f-b870-4e18-a51f-514432892aec\",\n \"name\": \"25287b3f-b870-4e18-a51f-514432892aec\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:31:45.913312Z\",\n \"endTime\": \"2026-09-29T00:00:52.759469958Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f16c375021ac3e568d45f076ae016cf4ba3ca1316b15e9eb5c118b9af2ad0e02","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"6cce0979bce816f94e1a76410fe9b0dfddfaeb9167c8d068f6c6c58b8957a7db","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a51a10\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mnwb4zfspsdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000a51a10\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-mnwb4zfspsdb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"fbc322678c5e06f92fab954b06af0fe8f945304394da5981fae2e46f7a32fea1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2104707837916614656","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"dd38e1bb9f1353d3c3ede4dc31e69fe80aa02069128581607c7d92391c522e32","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008a5e00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-282f9ccvqths/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0008a5e00\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-282f9ccvqths/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":14,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","79207d02cb7cb1faeeb39cb92f141efcce63951b8687d5bc3ee481ef2a4df1a8","83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc"],"matched_occurrences":14,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104670218818162688","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T20:30:57Z","started_at":"2026-09-28T20:30:57Z","completed_at":"2026-09-28T23:04:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T23:18:08.360052763Z","timing_metadata_checked_at":"2026-09-28T23:18:08.360052763Z","region":"brazilsouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T23:18:08.362785354Z","region_metadata_checked_at":"2026-09-28T23:18:08.362785354Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104669292057333760","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T20:27:16Z","started_at":"2026-09-28T20:27:16Z","completed_at":"2026-09-28T23:05:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T23:18:08.334512944Z","timing_metadata_checked_at":"2026-09-28T23:18:08.334512944Z","region":"westeurope","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T23:18:08.337946533Z","region_metadata_checked_at":"2026-09-28T23:18:08.337946533Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"54a7f49f5ea2b7d79828ae614a6830162841b03408761eb3510e192edaa0a2d8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104669292057333760","occurred_at":"2026-09-28T20:27:16Z","failed_at":"e2e","signature_id":"d239f5d1b83b70a7a970eaac57a5e3865875c1f179ca44cad09e2f730112d858","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00171b4a0\u003e: \n failed to create NodePool np-nmsq2l: failed waiting for nodepool=\"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\" in resourcegroup=\"rg-candidate-4-21-lf4lgs4pzp7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\",\n \"name\": \"8303805f-b000-457e-b724-d4df7eac6975\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T21:30:59.759776246Z\",\n \"endTime\": \"2026-09-28T21:50:00.145843425Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00171b4a0\u003e: \n failed to create NodePool np-nmsq2l: failed waiting for nodepool=\"np-nmsq2l\" for cluster \"cluster-candidate-4-21-nmsq2l\" in resourcegroup=\"rg-candidate-4-21-lf4lgs4pzp7j\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westeurope/hcpOperationStatuses/8303805f-b000-457e-b724-d4df7eac6975\",\n \"name\": \"8303805f-b000-457e-b724-d4df7eac6975\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T21:30:59.759776246Z\",\n \"endTime\": \"2026-09-28T21:50:00.145843425Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-21-lf4lgs4pzp7j-candidate-nmsq2l--managed/providers/Microsoft.Compute/virtualMachines/y5i6c0j1i3c3z4q-np-nmsq2l-ncgkp-j7cz8 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T20:21:29Z","started_at":"2026-09-28T20:21:29Z","completed_at":"2026-09-28T22:58:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T23:07:16.855751151Z","timing_metadata_checked_at":"2026-09-28T23:07:16.855751151Z","region":"eastus2","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T23:07:16.858621241Z","region_metadata_checked_at":"2026-09-28T23:07:16.858621241Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"23efdf86e42e91829cc1e82432f2e1321fc28ffbbb5b7751faa235c52a1337bb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","occurred_at":"2026-09-28T20:21:29Z","failed_at":"e2e","signature_id":"b06391e0cd724ab0d438d9c0420d007165f3fe823ffd52497e5d58364ca57ca5","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001139d00\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:88]: failed to select a VM size with ephemeral OS disk support; this typically indicates a SKU restriction or quota issue in the test subscription/region\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001139d00\u003e: \n selector \"ephemeral-osdisk-worker\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0","failure_pattern":"selector \"\u003cselector\u003e\" matched no usable VM size in eastus2: no unrestricted VM size satisfied the selector in the test location; this typically indicates a VM SKU restriction or quota issue with the test subscription/region, not a product defect","search_query":"occurred"}},{"row_id":"7b82c2406993609ab9d7ac74c1d07cde77674dc272b657170b3363d34f0e2332","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104667835979534336","occurred_at":"2026-09-28T20:21:29Z","failed_at":"e2e","signature_id":"0808cac9ab1e6eab75a259dd85739aff3ad9bf80be0b493ce8ad127f37249569","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8vn5sb\" for cluster \"cluster-candidate-4-20-8vn5sb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00081f3e0\u003e: \n failed to create NodePool np-8vn5sb: failed waiting for nodepool=\"np-8vn5sb\" for cluster \"cluster-candidate-4-20-8vn5sb\" in resourcegroup=\"rg-candidate-4-20-2kxj4bcd64pd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5b06b703-2ea9-4252-ab56-c1c6946ec407\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5b06b703-2ea9-4252-ab56-c1c6946ec407\",\n \"name\": \"5b06b703-2ea9-4252-ab56-c1c6946ec407\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T21:21:03.360782437Z\",\n \"endTime\": \"2026-09-28T21:40:07.046700763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-2kxj4bcd64pd-candidate-8vn5sb--managed/providers/Microsoft.Compute/virtualMachines/b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-2kxj4bcd64pd-candidate-8vn5sb--managed/providers/Microsoft.Compute/virtualMachines/b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-8vn5sb\" for cluster \"cluster-candidate-4-20-8vn5sb\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00081f3e0\u003e: \n failed to create NodePool np-8vn5sb: failed waiting for nodepool=\"np-8vn5sb\" for cluster \"cluster-candidate-4-20-8vn5sb\" in resourcegroup=\"rg-candidate-4-20-2kxj4bcd64pd\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5b06b703-2ea9-4252-ab56-c1c6946ec407\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2/hcpOperationStatuses/5b06b703-2ea9-4252-ab56-c1c6946ec407\",\n \"name\": \"5b06b703-2ea9-4252-ab56-c1c6946ec407\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T21:21:03.360782437Z\",\n \"endTime\": \"2026-09-28T21:40:07.046700763Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-2kxj4bcd64pd-candidate-8vn5sb--managed/providers/Microsoft.Compute/virtualMachines/b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-4-20-2kxj4bcd64pd-candidate-8vn5sb--managed/providers/Microsoft.Compute/virtualMachines/b8s1d9p4q0u8w7k-np-8vn5sb-x9xrv-l4tr9 to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["83e470f9393e9b09dda3e64b9f7050ad8494380a887be8f14ee34e24a885d95a","b467691e5db9f3897c4a90d9cd0f37d41365ea0195d8cf7200c1e9beba4738b0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104621627147816960","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T17:17:52Z","started_at":"2026-09-28T17:17:52Z","completed_at":"2026-09-28T19:29:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T19:36:05.325980711Z","timing_metadata_checked_at":"2026-09-28T19:36:05.325980711Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T19:36:05.330897695Z","region_metadata_checked_at":"2026-09-28T19:36:05.330897695Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104576811751968768","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T14:19:44Z","started_at":"2026-09-28T14:19:44Z","completed_at":"2026-09-28T16:44:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:58:44.666691228Z","timing_metadata_checked_at":"2026-09-28T16:58:44.666691228Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:58:44.66937642Z","region_metadata_checked_at":"2026-09-28T16:58:44.66937642Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"2f5e6c6ad65e2a0121ff114afd2b4a75e84b4bd52204ea4153b9ddf64ae49087","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104576811751968768","occurred_at":"2026-09-28T14:19:44Z","failed_at":"e2e","signature_id":"31429d7beca55879e992a9e74912d2072a12718cdce9ba4d4ebbfb9d70101e40","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0006d65e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1ed0a0-a90c-4672-a753-87b575007435\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1ed0a0-a90c-4672-a753-87b575007435\",\n \"name\": \"cf1ed0a0-a90c-4672-a753-87b575007435\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T14:48:00.539019012Z\",\n \"endTime\": \"2026-09-28T15:07:01.911525796Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-rhtj7-kpztn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tdccrn4dc9dh--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-rhtj7-kpztn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-rhtj7-kpztn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tdccrn4dc9dh--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-rhtj7-kpztn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:138]: failed to create node pool\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0006d65e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1ed0a0-a90c-4672-a753-87b575007435\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cf1ed0a0-a90c-4672-a753-87b575007435\",\n \"name\": \"cf1ed0a0-a90c-4672-a753-87b575007435\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T14:48:00.539019012Z\",\n \"endTime\": \"2026-09-28T15:07:01.911525796Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-rhtj7-kpztn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tdccrn4dc9dh--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-rhtj7-kpztn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-rhtj7-kpztn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-tdccrn4dc9dh--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-rhtj7-kpztn to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-tdccrn4dc9dh\" to finish creating: GET https://management"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104574026838970368","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T14:08:43Z","started_at":"2026-09-28T14:08:43Z","completed_at":"2026-09-28T16:46:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:58:44.53717422Z","timing_metadata_checked_at":"2026-09-28T16:58:44.53717422Z","region":"australiaeast","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:58:44.541824106Z","region_metadata_checked_at":"2026-09-28T16:58:44.541824106Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104573697271533568","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T14:07:25Z","started_at":"2026-09-28T14:07:25Z","completed_at":"2026-09-28T16:45:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:58:44.411694599Z","timing_metadata_checked_at":"2026-09-28T16:58:44.411694599Z","region":"centralindia","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:58:44.414248492Z","region_metadata_checked_at":"2026-09-28T16:58:44.414248492Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104573041903144960","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T14:04:48Z","started_at":"2026-09-28T14:04:48Z","completed_at":"2026-09-28T16:47:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T16:58:44.280748295Z","timing_metadata_checked_at":"2026-09-28T16:58:44.280748295Z","region":"switzerlandnorth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T16:58:44.284454484Z","region_metadata_checked_at":"2026-09-28T16:58:44.284454484Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2104520260127297536","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T10:35:04Z","started_at":"2026-09-28T10:35:04Z","completed_at":"2026-09-28T13:08:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T13:18:05.760444839Z","timing_metadata_checked_at":"2026-09-28T13:18:05.760444839Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T13:18:05.763564328Z","region_metadata_checked_at":"2026-09-28T13:18:05.763564328Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104390744759341056","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T02:00:25Z","started_at":"2026-09-28T02:00:25Z","completed_at":"2026-09-28T04:24:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T04:41:57.432237685Z","timing_metadata_checked_at":"2026-09-28T04:41:57.432237685Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T04:41:57.435441175Z","region_metadata_checked_at":"2026-09-28T04:41:57.435441175Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"88a70da9bc66a48eda9a9fe148a00700d48888bc71c8a672c3d835e48c7f12f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2104390744759341056","occurred_at":"2026-09-28T02:00:25Z","failed_at":"e2e","signature_id":"08f63884b8084a2606a1a5268b6fb65c1cc3e789a692cbd59f3ad65ea31fbc51","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4z25qr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7lft27tlvnbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0009fab20\u003e: \n storage account log verification failed\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox4z25qr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-7lft27tlvnbf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*errors.errorString | 0xc0009fab20\u003e: \n storage account log verification failed\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":2,"runs_with_occurrences":2,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":0,"runs_without_occurrence_rows":2,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T23:00:26Z","started_at":"2026-09-28T23:00:26Z","completed_at":"2026-09-29T01:25:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T01:35:23.859095948Z","timing_metadata_checked_at":"2026-09-29T01:35:23.859095948Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T01:35:23.862036238Z","region_metadata_checked_at":"2026-09-29T01:35:23.862036238Z"},"failed_at":["e2e"],"failed_test_count":13,"occurrences":[{"row_id":"072fb71f92136729640bc6e18f0c8952520e60ceeb0fa7a314a740a9ed76207f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"91d27f5f51891910ea04c7a1edc0380a08aadc419bba96bc422aa6887c894e1b","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000d235a8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000d235a8\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-28-094035, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"3ed39ace07e6cbbccb7dcd35c87945632c0c7e4752950b8672f325c82d6f1d8c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"6dc5b114f6ac687a7b4c1dd7c710e60fb5ddbfa17d78beb65f14479f54bce3fa","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001432880\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-lllf2h\" resourcegroup=\"rg-np-dg-minor-lllf2h-lmftzgmmkgfx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-lllf2h-lmftzgmmkgfx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-lllf2h/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:768]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001432880\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-lllf2h\" resourcegroup=\"rg-np-dg-minor-lllf2h-lmftzgmmkgfx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-lllf2h-lmftzgmmkgfx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-lllf2h/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"4e5a2479ac37037778805de37d4448355e976a330c196b2b4e144f27a4ad56a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"cbace1c1515d3a38f062cbe5eb63ec2ea5b5fee3036a914597181616e998cb1b","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dbaaa0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-grgfk2\" in resourcegroup=\"rg-np-version-upgrade-grgfk2-gjc5jfp7zsnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-grgfk2-gjc5jfp7zsnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-grgfk2/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000dbaaa0\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-grgfk2\" in resourcegroup=\"rg-np-version-upgrade-grgfk2-gjc5jfp7zsnb\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-grgfk2-gjc5jfp7zsnb/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-grgfk2/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"70bc8717bc37ddba31e03d088c8c3f5e52fb4aa40e7abf1fbcf57711a06802cd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"a7eff7ef2cb5084bb89a68bdcf135801052c5704577a8158bf99956be0c5976d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-vdf28j9mzrjf/cluster-nightly-5-1-ds89w5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a3e960\u003e: \n failed to create HCP cluster cluster-nightly-5-1-ds89w5: failed waiting for cluster=\"cluster-nightly-5-1-ds89w5\" in resourcegroup=\"rg-nightly-5-1-vdf28j9mzrjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cc98024-5165-40da-b846-ec5ee553964b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cc98024-5165-40da-b846-ec5ee553964b\",\n \"name\": \"4cc98024-5165-40da-b846-ec5ee553964b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:13:16.920404751Z\",\n \"endTime\": \"2026-09-29T00:42:21.556617536Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-vdf28j9mzrjf/cluster-nightly-5-1-ds89w5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a3e960\u003e: \n failed to create HCP cluster cluster-nightly-5-1-ds89w5: failed waiting for cluster=\"cluster-nightly-5-1-ds89w5\" in resourcegroup=\"rg-nightly-5-1-vdf28j9mzrjf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cc98024-5165-40da-b846-ec5ee553964b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4cc98024-5165-40da-b846-ec5ee553964b\",\n \"name\": \"4cc98024-5165-40da-b846-ec5ee553964b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:13:16.920404751Z\",\n \"endTime\": \"2026-09-29T00:42:21.556617536Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"7d76c96741c31493f0f043d3a37ce143da403c1396555ba4f5565c0ddca0545c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"13f4005d5995a342c9b357c20b4fe80d11b6f71b8daf1ca214978ae01677d844","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002a097a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-swk5ggttxwjr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002a097a0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-swk5ggttxwjr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"835367b735eccc68a87679e2977b2c7c4ac3cbc799a1325f9805e3b0667b1a50","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"3611ce8acc0c465680d47f7d670a7cddaae327fe71dbc3c6d5e7c236e72e2b9a","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00160fce0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hfkhgg4cj7jf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc00160fce0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-hfkhgg4cj7jf/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8dda6941915edab23c9458bef3eaae001ee6712b26fe76740aed7ed8fcb8ae63","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"eefa53d30dbe7c20530d9bfcbd3b414e69c13274d0f6976f0cd790c7047bfb7a","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014c1ec0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k8xstznc2dd6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014c1ec0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-k8xstznc2dd6/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9f4f5568b9951bb103832243d5c59b5a92966b8ebbfc9b688c13b58c499dc68e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"be520732f5a1c2ae3f44c1f3399ac933aa77be4a8bfb047291c2b5bfc9032219","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001873458\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-zwldz5957z9k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00c46f27-c5be-4ae1-883b-7e21c3ef06fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00c46f27-c5be-4ae1-883b-7e21c3ef06fc\",\n \"name\": \"00c46f27-c5be-4ae1-883b-7e21c3ef06fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:57:06.99965356Z\",\n \"endTime\": \"2026-09-29T00:16:13.233773533Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-lwwnr-xksft: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zwldz5957z9k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-lwwnr-xksft to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-lwwnr-xksft: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zwldz5957z9k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-lwwnr-xksft to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001873458\u003e: \n failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-zwldz5957z9k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00c46f27-c5be-4ae1-883b-7e21c3ef06fc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/00c46f27-c5be-4ae1-883b-7e21c3ef06fc\",\n \"name\": \"00c46f27-c5be-4ae1-883b-7e21c3ef06fc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T23:57:06.99965356Z\",\n \"endTime\": \"2026-09-29T00:16:13.233773533Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-lwwnr-xksft: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zwldz5957z9k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-lwwnr-xksft to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-lwwnr-xksft: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-zwldz5957z9k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-lwwnr-xksft to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a07afb5857786fe7cf475e152733051c024cc1cff85f8f24843d1095564af570","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"682bebdf23c8807964a6439128835338cd9f440c606c68367c05915320960318","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e91950\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-jnh726bgx45k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000e91950\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-jnh726bgx45k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"bd4907bdd0cadd68097eb42601534dd6c3a7cba31f699d785996a0a6115f148a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"44cfa63095c47519677515fcbb5edf1f6fef18f8fa9f804adde98dd248b04ee2","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b459e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-sf9f9x57dtfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000b459e0\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-sf9f9x57dtfl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"cf0e26e52e05a562d61f87ae1de12458db333070f62282eb669ebb02e92a94e0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"79be886538684366411edc9bf39d37cbb253c90860c0f3a74b84b86bf92ed16a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.0","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dpvlx8\" for cluster \"cluster-nightly-5-0-dpvlx8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012916a0\u003e: \n failed to create NodePool np-dpvlx8: failed waiting for nodepool=\"np-dpvlx8\" for cluster \"cluster-nightly-5-0-dpvlx8\" in resourcegroup=\"rg-nightly-5-0-5jrsqj92ljxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eee4d653-3b34-478d-9851-5151f132ff5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eee4d653-3b34-478d-9851-5151f132ff5e\",\n \"name\": \"eee4d653-3b34-478d-9851-5151f132ff5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:23:56.849718476Z\",\n \"endTime\": \"2026-09-29T00:43:01.619582837Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5jrsqj92ljxf-nightly-dpvlx8--managed/providers/Microsoft.Compute/virtualMachines/j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5jrsqj92ljxf-nightly-dpvlx8--managed/providers/Microsoft.Compute/virtualMachines/j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-dpvlx8\" for cluster \"cluster-nightly-5-0-dpvlx8\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012916a0\u003e: \n failed to create NodePool np-dpvlx8: failed waiting for nodepool=\"np-dpvlx8\" for cluster \"cluster-nightly-5-0-dpvlx8\" in resourcegroup=\"rg-nightly-5-0-5jrsqj92ljxf\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eee4d653-3b34-478d-9851-5151f132ff5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeadlineExceeded\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eee4d653-3b34-478d-9851-5151f132ff5e\",\n \"name\": \"eee4d653-3b34-478d-9851-5151f132ff5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-29T00:23:56.849718476Z\",\n \"endTime\": \"2026-09-29T00:43:01.619582837Z\",\n \"error\": {\n \"code\": \"DeadlineExceeded\",\n \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5jrsqj92ljxf-nightly-dpvlx8--managed/providers/Microsoft.Compute/virtualMachines/j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-5-0-5jrsqj92ljxf-nightly-dpvlx8--managed/providers/Microsoft.Compute/virtualMachines/j6a4t3c4n6z0s3d-np-dpvlx8-65kgg-vkztb to exist\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e","failure_pattern":"ERROR CODE: DeadlineExceeded; provider Microsoft.Compute","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"d729d25d86ab05ccee2140d277a5b15ccf3aa105b2da782ac5a0cb3cbcb34fcd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"938bb68fc81ec914d6ee9c7b65e3ee227bb2a191b4973a9f265b8fbb7db113af","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-br2c4x\" for cluster \"cluster-nightly-4-20-br2c4x\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008e6340\u003e: \n failed to create NodePool np-br2c4x: failed starting nodepool creation \"np-br2c4x\" for cluster \"cluster-nightly-4-20-br2c4x\" in resourcegroup=\"rg-nightly-4-20-bvth8x7dzstx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-bvth8x7dzstx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-br2c4x/nodePools/np-br2c4x\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-br2c4x\" for cluster \"cluster-nightly-4-20-br2c4x\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008e6340\u003e: \n failed to create NodePool np-br2c4x: failed starting nodepool creation \"np-br2c4x\" for cluster \"cluster-nightly-4-20-br2c4x\" in resourcegroup=\"rg-nightly-4-20-bvth8x7dzstx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-bvth8x7dzstx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-br2c4x/nodePools/np-br2c4x\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"ebc40728df217533f94326cfe0a9e34e2773b5cbc38f7f5ef7c73513c13adba9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2104707837958557696","occurred_at":"2026-09-28T23:00:26Z","failed_at":"e2e","signature_id":"d15cf8226866bf1b9e3e386a753681c173badd9fd0c5b9b634779ff33a37e3cc","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088a3c0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-7mm9jz\" in resourcegroup=\"rg-np-skip-minor-7mm9jz-mkfnfjw9mlgl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-7mm9jz-mkfnfjw9mlgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-7mm9jz/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:461]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-13-024541\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088a3c0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-7mm9jz\" in resourcegroup=\"rg-np-skip-minor-7mm9jz-mkfnfjw9mlgl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-7mm9jz-mkfnfjw9mlgl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-7mm9jz/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-13-024541\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":13,"failure_pattern_ids":["28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","aa708535d34459cf2acb6d995478e0a955e647c33f10d5894feafa25fa5c6699","b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","f41d66f38646e9401165dce74dc3ea622f837ea4799b21842b1058338204b62e"],"matched_occurrences":13,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2104687818398765056","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-28T21:40:53Z","started_at":"2026-09-28T21:40:53Z","completed_at":"2026-09-28T23:48:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-29T00:00:01.02644822Z","timing_metadata_checked_at":"2026-09-29T00:00:01.02644822Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-29T00:00:01.02971931Z","region_metadata_checked_at":"2026-09-29T00:00:01.02971931Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"fb382031c004cdb5c8e78d114bfee630ad3f6ca2af9057d1be1f92d3921348b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2104687818398765056","occurred_at":"2026-09-28T21:40:53Z","failed_at":"e2e","signature_id":"b9887fb5f4ff622cd6b40c348a1b64080f5877eb981c863198ae3705d8ec506f","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001152660\u003e: \n resource group \"rg-cp-ystream-upgrade-4-22-k576cn8fjplq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-c95x5l\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-k576cn8fjplq\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\",\n \"name\": \"5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T22:54:48.090437049Z\",\n \"endTime\": \"2026-09-28T23:18:56.801422937Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t49njjvu97eroh1p7j9jssq535c6590 still exists (deletion dispatched at 2026-09-28T22:54:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001152660\u003e: \n resource group \"rg-cp-ystream-upgrade-4-22-k576cn8fjplq\": failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cp-ystream-upgrade-4-22-c95x5l\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-k576cn8fjplq\" to finish deleting: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\",\n \"name\": \"5533b7af-fbcc-4912-9e0c-66a6d54c8b8b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-28T22:54:48.090437049Z\",\n \"endTime\": \"2026-09-28T23:18:56.801422937Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2t49njjvu97eroh1p7j9jssq535c6590 still exists (deletion dispatched at 2026-09-28T22:54:49Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["59fed0c2c672d88ecfaa1321b82b9471c7b2768e503a1d41c8cd9b5c92c10116"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2104437406588473344","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T05:05:50Z","started_at":"2026-09-28T05:05:50Z","completed_at":"2026-09-28T07:24:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T07:32:12.444655512Z","timing_metadata_checked_at":"2026-09-28T07:32:12.444655512Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T07:32:12.447699803Z","region_metadata_checked_at":"2026-09-28T07:32:12.447699803Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2104390744801284096","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-28T02:00:25Z","started_at":"2026-09-28T02:00:25Z","completed_at":"2026-09-28T03:54:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-28T04:00:08.43451136Z","timing_metadata_checked_at":"2026-09-28T04:00:08.43451136Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-28T04:00:08.43773205Z","region_metadata_checked_at":"2026-09-28T04:00:08.43773205Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}