{"meta":{"date":"2026-09-17","timezone":"UTC","generated_at":"2026-10-15T01:57:50Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":196,"failed_runs":176,"runs_with_occurrences":176,"runs_with_failure_pattern_matches":81,"runs_with_unmatched_occurrences":95,"runs_without_occurrence_rows":20,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100735274731442176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T23:54:53Z","started_at":"2026-09-17T23:54:53Z","completed_at":"2026-09-18T01:49:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:45.243153167Z","timing_metadata_checked_at":"2026-09-18T15:40:45.243153167Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:45.246394578Z","region_metadata_checked_at":"2026-09-18T15:40:45.246394578Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9525a70b87461b6510089a443f8e92b560e266159be056e5ab36af224a417f33","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100735274731442176","occurred_at":"2026-09-17T23:54:53Z","failed_at":"e2e","signature_id":"a9eee6e574575c9254a566a1c15978b6cdd24af6e7e61c298d051c731edd627c","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rrwf84jdrq9t/cluster-hs-pre-dlzcc5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007ce420\u003e: \n failed to create HCP cluster cluster-hs-pre-dlzcc5: failed waiting for cluster=\"cluster-hs-pre-dlzcc5\" in resourcegroup=\"rg-hs-pre--rrwf84jdrq9t\" to finish creating: GET https://rp.j1442176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rrwf84jdrq9t/cluster-hs-pre-dlzcc5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007ce420\u003e: \n failed to create HCP cluster cluster-hs-pre-dlzcc5: failed waiting for cluster=\"cluster-hs-pre-dlzcc5\" in resourcegroup=\"rg-hs-pre--rrwf84jdrq9t\" to finish creating: GET https://rp.j1442176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100734811915161600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6990,"pr_state":"closed","pr_sha":"a0902051078fd0481b53edb539f46458fe329aca","final_merged_sha":"a0902051078fd0481b53edb539f46458fe329aca","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T23:53:03Z","started_at":"2026-09-17T23:53:03Z","completed_at":"2026-09-18T01:58:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:14.564178566Z","timing_metadata_checked_at":"2026-09-18T15:41:14.564178566Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:14.579843522Z","region_metadata_checked_at":"2026-09-18T15:41:14.579843522Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721153071910912","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6998,"pr_state":"closed","pr_sha":"c947fb99750b0318d83264c65a0eebdbade0c5e1","final_merged_sha":"c947fb99750b0318d83264c65a0eebdbade0c5e1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T22:58:39Z","started_at":"2026-09-17T22:58:39Z","completed_at":"2026-09-18T01:15:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:18.111770999Z","timing_metadata_checked_at":"2026-09-18T15:41:18.111770999Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:18.115175911Z","region_metadata_checked_at":"2026-09-18T15:41:18.115175911Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721125368532992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T22:58:38Z","started_at":"2026-09-17T22:58:38Z","completed_at":"2026-09-18T01:00:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:56.721982653Z","timing_metadata_checked_at":"2026-09-18T15:40:56.721982653Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:56.725054864Z","region_metadata_checked_at":"2026-09-18T15:40:56.725054864Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"e8c9d31f20a986152550d7eeefa919b4a9f3335e39de7be900c361787999b254","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721125368532992","occurred_at":"2026-09-17T22:58:38Z","failed_at":"e2e","signature_id":"3ad0c7f8513c2d305c0610677386a9460eaa446d3f7dfb59c408d79cec65a7ec","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001d94018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-g7cwbl5k67j4\" to finish creating: GET https://rp.j8532992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n \"name\": \"fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T00:05:39.3471997Z\",\n \"endTime\": \"2026-09-18T00:06:19.6272248Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n \u003c*errors.joinError | 0xc001d94018\u003e: \n VerifyNodesReady failed: no nodes found in the cluster\n VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-g7cwbl5k67j4\" to finish creating: GET https://rp.j8532992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n \"name\": \"fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-18T00:05:39.3471997Z\",\n \"endTime\": \"2026-09-18T00:06:19.6272248Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721105009381376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6922,"pr_state":"open","pr_sha":"87e40a07142a1457ed3f83374cc5d085c87d8f29","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T22:58:35Z","started_at":"2026-09-17T22:58:35Z","completed_at":"2026-09-18T00:59:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:40.716507776Z","timing_metadata_checked_at":"2026-09-18T15:40:40.716507776Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:40.720046589Z","region_metadata_checked_at":"2026-09-18T15:40:40.720046589Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"4c496920393ebe9f4b90fab311b385d0f1b40c15857b3397b41e4e18cb11b129","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721105009381376","occurred_at":"2026-09-17T22:58:35Z","failed_at":"alert","signature_id":"82e0a12c7cd95fd907f63543eeac97bdeae8d5f9e5a424bdde230db4ef5d4c57","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-18T00:02:16Z\n Ended: 2026-09-18T00:27:14Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9381376-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st2tc78t6cvdchmjv242i9bdc8rpm9q-k2h4w6t5x7l3o9z\", pod=\"router-586c95bb7f-lzcz4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st2tc78t6cvdchmjv242i9bdc8rpm9q-k2h4w6t5x7l3o9z/router-586c95bb7f-lzcz4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c0cdf8fcb1bfe2bb34569d3f104afa263461b584fac75a5aa3f1851687b5730e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721105009381376","occurred_at":"2026-09-17T22:58:35Z","failed_at":"e2e","signature_id":"d601eba3abc4f82457501b40b90cc5d7561cc3383a0ad74d5298ef8e79a87795","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f703e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-lxp82nsl4llc\" to finish creating: GET https://rp.j9381376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9411175-a2f7-4b2d-9324-66fd2acc379f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9411175-a2f7-4b2d-9324-66fd2acc379f\",\n \"name\": \"f9411175-a2f7-4b2d-9324-66fd2acc379f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:50:07.442410011Z\",\n \"endTime\": \"2026-09-18T00:19:13.547086739Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f703e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-lxp82nsl4llc\" to finish creating: GET https://rp.j9381376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9411175-a2f7-4b2d-9324-66fd2acc379f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9411175-a2f7-4b2d-9324-66fd2acc379f\",\n \"name\": \"f9411175-a2f7-4b2d-9324-66fd2acc379f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:50:07.442410011Z\",\n \"endTime\": \"2026-09-18T00:19:13.547086739Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721091256258560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"ea503ec41ff47705c9a97cd58f7750bc3a983568","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T22:58:32Z","started_at":"2026-09-17T22:58:32Z","completed_at":"2026-09-18T01:22:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.212036027Z","timing_metadata_checked_at":"2026-09-18T15:41:21.212036027Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:21.218615851Z","region_metadata_checked_at":"2026-09-18T15:41:21.218615851Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"48548e0622dc040cfb45dbf2da8b5024c9623e553eb93317d7c35136b851b213","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721091256258560","occurred_at":"2026-09-17T22:58:32Z","failed_at":"alert","signature_id":"fffbb7c99a9e3fbfbbfb27296f90fe1eaf381e133c47c2097dc7c5b3440df9c4","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-18T01:12:37Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6258560-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-m8n67\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-m8n67 (namespace arobit) on cluster ci01-j6258560-svc has averaged 125.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721085266792448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6988,"pr_state":"closed","pr_sha":"53b8fd9bee7c04069b9537ac28ccedffde355125","final_merged_sha":"53b8fd9bee7c04069b9537ac28ccedffde355125","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T22:58:30Z","started_at":"2026-09-17T22:58:30Z","completed_at":"2026-09-18T01:14:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.248573996Z","timing_metadata_checked_at":"2026-09-18T15:41:12.248573996Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:12.25257361Z","region_metadata_checked_at":"2026-09-18T15:41:12.25257361Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721035019030528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"3f0521ff633af0368f049bfead3093e31565b50f","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T22:58:18Z","started_at":"2026-09-17T22:58:18Z","completed_at":"2026-09-17T23:01:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:38.495316682Z","timing_metadata_checked_at":"2026-09-18T15:40:38.495316682Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:38.499390296Z","region_metadata_checked_at":"2026-09-19T03:30:36.373674936Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6023a2bd51968631652f9d5e596c270d3a24b6ad44cb1fe7d41b4509245ba5c7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721035019030528","occurred_at":"2026-09-17T22:58:18Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721018606718976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T22:58:13Z","started_at":"2026-09-17T22:58:13Z","completed_at":"2026-09-18T01:08:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:46.847060036Z","timing_metadata_checked_at":"2026-09-18T15:40:46.847060036Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:46.851245151Z","region_metadata_checked_at":"2026-09-18T15:40:46.851245151Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100714303135944704","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6880,"pr_state":"closed","pr_sha":"7debc556d15879cf7cc9ac5e18c28db269e245f8","final_merged_sha":"a20dd280dd6b7499cf853051219487314ef1e460","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T22:31:33Z","started_at":"2026-09-17T22:31:33Z","completed_at":"2026-09-18T00:29:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:31.230015847Z","timing_metadata_checked_at":"2026-09-18T15:40:31.230015847Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:31.23338396Z","region_metadata_checked_at":"2026-09-18T15:40:31.23338396Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1f513663dfe40abf177d7d7193592552375bb08c08c6ca93c50a4838105679bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100714303135944704","occurred_at":"2026-09-17T22:31:33Z","failed_at":"e2e","signature_id":"7468e248b9c77a11eebc376f236b7610c67fe482e15792bf83562366e5934d5f","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007926a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vsg6jbtlvgqx\" to finish creating: GET https://rp.j5944704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\",\n \"name\": \"ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:20:00.021017752Z\",\n \"endTime\": \"2026-09-17T23:49:08.836539572Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007926a0\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vsg6jbtlvgqx\" to finish creating: GET https://rp.j5944704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\",\n \"name\": \"ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:20:00.021017752Z\",\n \"endTime\": \"2026-09-17T23:49:08.836539572Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713962864644096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T22:30:12Z","started_at":"2026-09-17T22:30:12Z","completed_at":"2026-09-18T00:41:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:08.477696568Z","timing_metadata_checked_at":"2026-09-18T15:41:08.477696568Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:08.481201381Z","region_metadata_checked_at":"2026-09-18T15:41:08.481201381Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713215490002944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T22:27:14Z","started_at":"2026-09-17T22:27:14Z","completed_at":"2026-09-18T00:37:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:33.105982676Z","timing_metadata_checked_at":"2026-09-18T15:41:33.105982676Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:33.109603989Z","region_metadata_checked_at":"2026-09-18T15:41:33.109603989Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3d4835088af80bf59e91f550e719ef1ae4985e975bb96ff352ab0534b65bf179","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713215490002944","occurred_at":"2026-09-17T22:27:14Z","failed_at":"e2e","signature_id":"db652501caa7a4389b880b5afa48aab924fa2baf6e02836e6489573d8b826dbb","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006f07e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6f9wkzp5v4jd\" to finish creating: GET https://rp.j0002944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97dac56f-020e-4301-ba21-2ca0fd4e2f6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97dac56f-020e-4301-ba21-2ca0fd4e2f6d\",\n \"name\": \"97dac56f-020e-4301-ba21-2ca0fd4e2f6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:07.249708545Z\",\n \"endTime\": \"2026-09-17T23:58:12.85966363Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m18s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006f07e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-6f9wkzp5v4jd\" to finish creating: GET https://rp.j0002944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97dac56f-020e-4301-ba21-2ca0fd4e2f6d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/97dac56f-020e-4301-ba21-2ca0fd4e2f6d\",\n \"name\": \"97dac56f-020e-4301-ba21-2ca0fd4e2f6d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:07.249708545Z\",\n \"endTime\": \"2026-09-17T23:58:12.85966363Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m18s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"616d7cc935b1e16b3abddebf365cc7e89e6066f0ba0ecb467e029b2f176133f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713215490002944","occurred_at":"2026-09-17T22:27:14Z","failed_at":"e2e","signature_id":"b9f0cd575efa9d87456c559803742a6d89fb91600ce45815cecb4689bcc702f8","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe8980\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hgg4wl6sg8sc\" to finish creating: GET https://rp.j0002944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f65c2fa-1e9a-4309-96ea-66a270395b00\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f65c2fa-1e9a-4309-96ea-66a270395b00\",\n \"name\": \"2f65c2fa-1e9a-4309-96ea-66a270395b00\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:30:18.236986606Z\",\n \"endTime\": \"2026-09-17T23:59:22.774158112Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe8980\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hgg4wl6sg8sc\" to finish creating: GET https://rp.j0002944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f65c2fa-1e9a-4309-96ea-66a270395b00\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f65c2fa-1e9a-4309-96ea-66a270395b00\",\n \"name\": \"2f65c2fa-1e9a-4309-96ea-66a270395b00\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:30:18.236986606Z\",\n \"endTime\": \"2026-09-17T23:59:22.774158112Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712898639695872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T22:25:59Z","started_at":"2026-09-17T22:25:59Z","completed_at":"2026-09-18T00:32:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:59.247708837Z","timing_metadata_checked_at":"2026-09-18T15:40:59.247708837Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:59.253622459Z","region_metadata_checked_at":"2026-09-18T15:40:59.253622459Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"b75ecc94cd12eadfe8794753ff953ecb2fd94714fb63598cacce34acaaf7857b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712898639695872","occurred_at":"2026-09-17T22:25:59Z","failed_at":"alert","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T23:37:25Z\n Ended: 2026-09-17T23:53:21Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9695872-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T23:46:22Z\n Ended: 2026-09-17T23:56:22Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9695872-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"8bccd883802213d6f2eaa418fd8acd9b2cd4c3fa","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T22:25:29Z","started_at":"2026-09-17T22:25:29Z","completed_at":"2026-09-18T00:38:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.172620863Z","timing_metadata_checked_at":"2026-09-18T15:41:20.172620863Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:20.175787374Z","region_metadata_checked_at":"2026-09-18T15:41:20.175787374Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"0ccecc98d2fd473657b412d4b33569b51d5411d22b66c9acc11e33edb7407bfa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","failed_at":"e2e","signature_id":"3952b4207ca326cafd047aae37e1541b9c1921b3b3a1de670914b6430814079b","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000648280\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qghsblcj4pgk\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n \"name\": \"f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:11.075687263Z\",\n \"endTime\": \"2026-09-17T23:58:14.591633856Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000648280\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qghsblcj4pgk\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n \"name\": \"f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:11.075687263Z\",\n \"endTime\": \"2026-09-17T23:58:14.591633856Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"33ef816299478b196b036cf16e240552698c6b28421d8906e17ba5b6ea8bd969","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","failed_at":"alert","signature_id":"1033d58df80ee9a7d767bdc19425ad417e226db41f4f4ad1a7aa6ed4d2688d8a","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 9 time(s) (3 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T23:40:23Z\n Ended: 2026-09-18T00:06:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4009472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st2jdcitjtn6r1jl4q34p61lesqvar0-l2u3i7e5p7i0m7j\", pod=\"router-dd88b46c7-ckx8f\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st2jdcitjtn6r1jl4q34p61lesqvar0-l2u3i7e5p7i0m7j/router-dd88b46c7-ckx8f has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T23:42:25Z\n Ended: 2026-09-18T00:11:23Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4009472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st2ji98etmkm3fuvovie18d64b0rt6v-v20260901\", pod=\"router-7fd9f9b4c6-gn2vk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st2ji98etmkm3fuvovie18d64b0rt6v-v20260901/router-7fd9f9b4c6-gn2vk has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T23:42:25Z\n Ended: 2026-09-18T00:06:22Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4009472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st2ji98etmkm3fuvovie18d64b0rt6v-v20260901\", pod=\"router-7fd9f9b4c6-q4q5l\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st2ji98etmkm3fuvovie18d64b0rt6v-v20260901/router-7fd9f9b4c6-q4q5l has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"367107959de7e78a5cebedb06fe818a8b0a3a577917be6ffd3be2758d4d8421f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","failed_at":"e2e","signature_id":"d74cdb3a0399ea27cc6c2ee2e3e86b457bc6ff248822afc15a1f630792589716","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-5xnxlfwdsctw/cluster-candidate-4-22-9lfv7s should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edc880\u003e: \n failed to create HCP cluster cluster-candidate-4-22-9lfv7s: failed waiting for cluster=\"cluster-candidate-4-22-9lfv7s\" in resourcegroup=\"rg-candidate-4-22-5xnxlfwdsctw\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0b39836-fc96-480c-9689-6141753bae0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0b39836-fc96-480c-9689-6141753bae0a\",\n \"name\": \"c0b39836-fc96-480c-9689-6141753bae0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:11.159787793Z\",\n \"endTime\": \"2026-09-17T23:58:14.737814238Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-5xnxlfwdsctw/cluster-candidate-4-22-9lfv7s should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000edc880\u003e: \n failed to create HCP cluster cluster-candidate-4-22-9lfv7s: failed waiting for cluster=\"cluster-candidate-4-22-9lfv7s\" in resourcegroup=\"rg-candidate-4-22-5xnxlfwdsctw\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0b39836-fc96-480c-9689-6141753bae0a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0b39836-fc96-480c-9689-6141753bae0a\",\n \"name\": \"c0b39836-fc96-480c-9689-6141753bae0a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:11.159787793Z\",\n \"endTime\": \"2026-09-17T23:58:14.737814238Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c73571d7b9a215f20fba0d78825f4c12ac9be2ff8be6be882692917a7fc68e4c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","failed_at":"e2e","signature_id":"7bfd2cf527b8d88bab96af9ab4da562292d265e26e2314dcdcf6bffc6b100ac5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-6pj6568ghzb2/cluster-candidate-4-21-k6bndh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc220\u003e: \n failed to create HCP cluster cluster-candidate-4-21-k6bndh: failed waiting for cluster=\"cluster-candidate-4-21-k6bndh\" in resourcegroup=\"rg-candidate-4-21-6pj6568ghzb2\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3d3840e-8750-44f9-b1f2-00d9d2c11a0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3d3840e-8750-44f9-b1f2-00d9d2c11a0e\",\n \"name\": \"f3d3840e-8750-44f9-b1f2-00d9d2c11a0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:11.265897654Z\",\n \"endTime\": \"2026-09-17T23:58:14.739219867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m6s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-6pj6568ghzb2/cluster-candidate-4-21-k6bndh should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc220\u003e: \n failed to create HCP cluster cluster-candidate-4-21-k6bndh: failed waiting for cluster=\"cluster-candidate-4-21-k6bndh\" in resourcegroup=\"rg-candidate-4-21-6pj6568ghzb2\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3d3840e-8750-44f9-b1f2-00d9d2c11a0e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3d3840e-8750-44f9-b1f2-00d9d2c11a0e\",\n \"name\": \"f3d3840e-8750-44f9-b1f2-00d9d2c11a0e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:11.265897654Z\",\n \"endTime\": \"2026-09-17T23:58:14.739219867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m6s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d7dbf04087404a7898381ade831cae9436e369b787863b0732f83bbc492b1833","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","failed_at":"e2e","signature_id":"6dff3d6d762cfe52b458282a2d61a5824494a4b7fbf9f3aab1ba2e26d05698c5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-k8zkqfwswxpz/cluster-candidate-4-20-z4ljdv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efc7c0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-z4ljdv: failed waiting for cluster=\"cluster-candidate-4-20-z4ljdv\" in resourcegroup=\"rg-candidate-4-20-k8zkqfwswxpz\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/290c11b0-aade-4173-8aad-dda5c860485f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/290c11b0-aade-4173-8aad-dda5c860485f\",\n \"name\": \"290c11b0-aade-4173-8aad-dda5c860485f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:11.979295423Z\",\n \"endTime\": \"2026-09-17T23:58:14.730283695Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 22m5s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-k8zkqfwswxpz/cluster-candidate-4-20-z4ljdv should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000efc7c0\u003e: \n failed to create HCP cluster cluster-candidate-4-20-z4ljdv: failed waiting for cluster=\"cluster-candidate-4-20-z4ljdv\" in resourcegroup=\"rg-candidate-4-20-k8zkqfwswxpz\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/290c11b0-aade-4173-8aad-dda5c860485f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/290c11b0-aade-4173-8aad-dda5c860485f\",\n \"name\": \"290c11b0-aade-4173-8aad-dda5c860485f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:29:11.979295423Z\",\n \"endTime\": \"2026-09-17T23:58:14.730283695Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 22m5s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e6d69bc3bfee287d6cb978662dafc5470f41c2fdeead9935347f8e882b292ee4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","failed_at":"e2e","signature_id":"e593b0bc62b406e1f219043a4233fe6d4b2427734c6df50023d50f3222509f40","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f70c60\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wkvscvwv244k\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d69226f-9a2e-4afc-ae7f-94f222fc3a40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d69226f-9a2e-4afc-ae7f-94f222fc3a40\",\n \"name\": \"4d69226f-9a2e-4afc-ae7f-94f222fc3a40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:28:52.213252998Z\",\n \"endTime\": \"2026-09-17T23:57:54.693039246Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f70c60\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-wkvscvwv244k\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d69226f-9a2e-4afc-ae7f-94f222fc3a40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4d69226f-9a2e-4afc-ae7f-94f222fc3a40\",\n \"name\": \"4d69226f-9a2e-4afc-ae7f-94f222fc3a40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:28:52.213252998Z\",\n \"endTime\": \"2026-09-17T23:57:54.693039246Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712492593319936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T22:24:22Z","started_at":"2026-09-17T22:24:22Z","completed_at":"2026-09-18T00:37:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:03.163998734Z","timing_metadata_checked_at":"2026-09-18T15:41:03.163998734Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:03.168056448Z","region_metadata_checked_at":"2026-09-18T15:41:03.168056448Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100707805496348672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6986,"pr_state":"closed","pr_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","final_merged_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T22:05:44Z","started_at":"2026-09-17T22:05:44Z","completed_at":"2026-09-18T00:01:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:11.739848557Z","timing_metadata_checked_at":"2026-09-18T15:41:11.739848557Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:11.743236769Z","region_metadata_checked_at":"2026-09-18T15:41:11.743236769Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100705809024421888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"344450021ef1cd34320edd7790bea47a65b8b30e","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-17T21:57:48Z","started_at":"2026-09-17T21:57:48Z","completed_at":"2026-09-17T23:52:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:53.91584916Z","timing_metadata_checked_at":"2026-09-18T15:40:53.91584916Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:53.918860671Z","region_metadata_checked_at":"2026-09-18T15:40:53.918860671Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100701574740316160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6805,"pr_state":"closed","pr_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","final_merged_sha":"b3bec0e87c8b5d73d493df4173e57e3c3590d2d7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T21:40:59Z","started_at":"2026-09-17T21:40:59Z","completed_at":"2026-09-17T22:44:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:24.630018205Z","timing_metadata_checked_at":"2026-09-18T15:40:24.630018205Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:24.633345117Z","region_metadata_checked_at":"2026-09-18T15:40:24.633345117Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"573ecc1d3c08f0c17d31c608916c7e78a633e36a1bbc380aabbaaca61fee9b94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100701574740316160","occurred_at":"2026-09-17T21:40:59Z","failed_at":"provision","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-17T22:08:23.042Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T22:08:24.010Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-17T22:08:24.819Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=949ef12d6ab429b901225672cf02dc8c2e8f5344e8e4f364f694e8721bc1ed4e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0316160-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F949ef12d6ab429b901225672cf02dc8c2e8f5344e8e4f364f694e8721bc1ed4e\ntime=2026-09-17T22:38:22.805Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100699515785842688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7002,"pr_state":"closed","pr_sha":"10032c5b3761591fce378756ccf11169001421b8","final_merged_sha":"8aa8c5953b4447b53d7b7c0ed058c6ed2c1be014","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T21:32:45Z","started_at":"2026-09-17T21:32:45Z","completed_at":"2026-09-17T23:36:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.662105857Z","timing_metadata_checked_at":"2026-09-18T15:41:21.662105857Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:21.666577473Z","region_metadata_checked_at":"2026-09-18T15:41:21.666577473Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"1c2af62d661ccbdadd3f984b9c3a3e5349279eae0bb9f123fd4a65bc6f581168","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100699515785842688","occurred_at":"2026-09-17T21:32:45Z","failed_at":"alert","signature_id":"18121697c147e2ac50157b93c7f4175f042996dac56448b883bc040150ef3bb5","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T22:37:27Z\n Ended: 2026-09-17T22:43:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5842688-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st1lu1gnkusrr9aaeo128agqit9e6jo-a8y8c4i0m5j1k4s\", pod=\"router-6f4d998d46-d2xxt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st1lu1gnkusrr9aaeo128agqit9e6jo-a8y8c4i0m5j1k4s/router-6f4d998d46-d2xxt has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T22:37:27Z\n Ended: 2026-09-17T22:43:25Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5842688-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st1lcku7i9kn16epu26j9jmha4t9219-t5q0q7d1s0b0b2h\", pod=\"router-5b9f46c564-vq9rt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st1lcku7i9kn16epu26j9jmha4t9219-t5q0q7d1s0b0b2h/router-5b9f46c564-vq9rt has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"open","pr_sha":"6e7413f9261ae3f89f1d8b9b4817769e713e32da","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T21:25:44Z","started_at":"2026-09-17T21:25:44Z","completed_at":"2026-09-17T23:18:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:01.201989569Z","timing_metadata_checked_at":"2026-09-18T15:41:01.201989569Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:01.205001279Z","region_metadata_checked_at":"2026-09-18T15:41:01.205001279Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"1f5ec65fb01b9f2d23a702acb092cc386c95bb5152ffb2e15efcfe4482d76559","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","failed_at":"e2e","signature_id":"78fecc2305e80f61dc9910998cc7c318760f78b41aed63eb7f46b3ce837e0d9f","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007b82c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pd9rhzzfgcc5\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aec160c6-5a35-495d-bd6d-5268450f3305\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aec160c6-5a35-495d-bd6d-5268450f3305\",\n \"name\": \"aec160c6-5a35-495d-bd6d-5268450f3305\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:12:49.625987517Z\",\n \"endTime\": \"2026-09-17T22:41:55.499178233Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007b82c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-pd9rhzzfgcc5\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aec160c6-5a35-495d-bd6d-5268450f3305\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aec160c6-5a35-495d-bd6d-5268450f3305\",\n \"name\": \"aec160c6-5a35-495d-bd6d-5268450f3305\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:12:49.625987517Z\",\n \"endTime\": \"2026-09-17T22:41:55.499178233Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"45eeb38e9d8ddea24da63372c463a498b64fb9e1c37e0de333079ee9c2430e5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","failed_at":"e2e","signature_id":"b07610b43263bb7e953aa2613bf43c66ef66f67a6d400930f3acc7d9b2871dfc","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f68560\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-6t2khml5tznz\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/137a17eb-a935-4a81-8849-b3928c62acad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/137a17eb-a935-4a81-8849-b3928c62acad\",\n \"name\": \"137a17eb-a935-4a81-8849-b3928c62acad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:14:35.599602035Z\",\n \"endTime\": \"2026-09-17T22:43:38.546147Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f68560\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-6t2khml5tznz\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/137a17eb-a935-4a81-8849-b3928c62acad\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/137a17eb-a935-4a81-8849-b3928c62acad\",\n \"name\": \"137a17eb-a935-4a81-8849-b3928c62acad\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:14:35.599602035Z\",\n \"endTime\": \"2026-09-17T22:43:38.546147Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5389be4d0273dfb3eaf0798f3e00dea14a8d809de32c8f1facc26b30fb0d1f41","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","failed_at":"e2e","signature_id":"aef26cedd47d167588772783661deaa26628c5b57bcb923a1a0a520ea749663a","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1a2a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-x6272vqbqkct\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84a3014a-22f5-4e28-9c7d-08c6235f1490\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84a3014a-22f5-4e28-9c7d-08c6235f1490\",\n \"name\": \"84a3014a-22f5-4e28-9c7d-08c6235f1490\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:13:03.512574477Z\",\n \"endTime\": \"2026-09-17T22:42:05.645433373Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m36s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1a2a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-x6272vqbqkct\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84a3014a-22f5-4e28-9c7d-08c6235f1490\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84a3014a-22f5-4e28-9c7d-08c6235f1490\",\n \"name\": \"84a3014a-22f5-4e28-9c7d-08c6235f1490\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:13:03.512574477Z\",\n \"endTime\": \"2026-09-17T22:42:05.645433373Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m36s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"85b717057caa4f27805d14846e0c2550306d6c8673d2f2ebae11bc1fc4ca9789","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","failed_at":"e2e","signature_id":"50d86109b369a86d21ce7f771945b06926a4c38d2d49433ad0d362386d5132f7","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ba240\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6jfrxnr5w7hl\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6f5865b-82eb-499b-9d49-57742af03c1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6f5865b-82eb-499b-9d49-57742af03c1b\",\n \"name\": \"b6f5865b-82eb-499b-9d49-57742af03c1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:12:49.653858883Z\",\n \"endTime\": \"2026-09-17T22:41:55.67204599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010ba240\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-6jfrxnr5w7hl\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6f5865b-82eb-499b-9d49-57742af03c1b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b6f5865b-82eb-499b-9d49-57742af03c1b\",\n \"name\": \"b6f5865b-82eb-499b-9d49-57742af03c1b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:12:49.653858883Z\",\n \"endTime\": \"2026-09-17T22:41:55.67204599Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ca1053549227f473351a7eb44ef7c89a1f5da13754735bbce1acd3b0150eb959","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","failed_at":"alert","signature_id":"d2aa839fc01483bc5381babfe51639f5c2325311fddfd8c5d39caa32de46dc09","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s) (4 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T22:25:10Z\n Ended: 2026-09-17T22:54:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5800576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st1fogv0fkmdfbch8spar202qsf3fdi-v20260901\", pod=\"router-56bd6ddfc5-bmvj7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st1fogv0fkmdfbch8spar202qsf3fdi-v20260901/router-56bd6ddfc5-bmvj7 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T22:26:09Z\n Ended: 2026-09-17T22:53:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5800576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st1fsfr24pblv7ln1lp377gfrc09jmf-v20261001\", pod=\"router-68476f767d-wdknb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st1fsfr24pblv7ln1lp377gfrc09jmf-v20261001/router-68476f767d-wdknb has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T22:26:09Z\n Ended: 2026-09-17T22:36:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5800576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st1fov1spppgn6o9h5ptch95gtec7bo-ingress-cert-le\", pod=\"router-7f74c869f6-n6zvj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st1fov1spppgn6o9h5ptch95gtec7bo-ingress-cert-le/router-7f74c869f6-n6zvj has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T22:26:09Z\n Ended: 2026-09-17T22:53:11Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5800576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st1fteggef19rbhrbirc9jgdq6conlt-w0k7b1n5e9p9u0h\", pod=\"router-7f679584bc-qz5ts\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st1fteggef19rbhrbirc9jgdq6conlt-w0k7b1n5e9p9u0h/router-7f679584bc-qz5ts has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"cefe0261105ebdcbb39ffdceec09832edf0b525ad39bdc3bd345c6a3f3ea69d2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","failed_at":"e2e","signature_id":"0a1d46d442635db3b7061c82189b8035af60a48310a53ecc646e31620910a1cb","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:91]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a84e0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-tdqjfrqdmsbh\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/241526e5-f5bb-4ce5-9863-b91ecf53191b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/241526e5-f5bb-4ce5-9863-b91ecf53191b\",\n \"name\": \"241526e5-f5bb-4ce5-9863-b91ecf53191b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:13:08.316752119Z\",\n \"endTime\": \"2026-09-17T22:42:15.586114195Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m46s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:91]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009a84e0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-tdqjfrqdmsbh\" to finish creating: GET https://rp.j5800576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/241526e5-f5bb-4ce5-9863-b91ecf53191b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/241526e5-f5bb-4ce5-9863-b91ecf53191b\",\n \"name\": \"241526e5-f5bb-4ce5-9863-b91ecf53191b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:13:08.316752119Z\",\n \"endTime\": \"2026-09-17T22:42:15.586114195Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m46s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100696985500651520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T21:22:45Z","started_at":"2026-09-17T21:22:45Z","completed_at":"2026-09-17T21:23:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:03.034686068Z","timing_metadata_checked_at":"2026-09-18T15:41:03.034686068Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:03.038814983Z","region_metadata_checked_at":"2026-09-19T03:30:39.553120548Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"36dfa76fbaea742b02773a33486746d136444331f7d6f55f33595ba700a76b7e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100696985500651520","occurred_at":"2026-09-17T21:22:45Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100694954127921152","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5562,"pr_state":"closed","pr_sha":"e0440b68b80a986602a8d6b89cd125f3b510e8b8","final_merged_sha":"18cce3707ed7df30b1a12d9779a8035dbdc24710","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-17T21:14:40Z","started_at":"2026-09-17T21:14:40Z","completed_at":"2026-09-17T23:06:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:02.265399027Z","timing_metadata_checked_at":"2026-09-18T15:40:02.265399027Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:02.268765039Z","region_metadata_checked_at":"2026-09-18T15:40:02.268765039Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691156575719424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T20:59:35Z","started_at":"2026-09-17T20:59:35Z","completed_at":"2026-09-17T22:56:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:45.079233877Z","timing_metadata_checked_at":"2026-09-18T15:40:45.079233877Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:45.082582489Z","region_metadata_checked_at":"2026-09-18T15:40:45.082582489Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"c31a6c58683edf16c970bbb8e2ba8a7ad4201e7915d320ad88f19c040116dfb8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691156575719424","occurred_at":"2026-09-17T20:59:35Z","failed_at":"e2e","signature_id":"13adef91a90ab725dab440fb0cd3d153b3bf6f23ccf1898e5f571b9c23787d80","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:124]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c52c40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-s2zdnnlcz4wt\" to finish creating: GET https://rp.j5719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80008e-4b66-48be-b60f-ca94f871de32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80008e-4b66-48be-b60f-ca94f871de32\",\n \"name\": \"1a80008e-4b66-48be-b60f-ca94f871de32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:56:32.579892703Z\",\n \"endTime\": \"2026-09-17T22:15:39.575240678Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:124]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c52c40\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-s2zdnnlcz4wt\" to finish creating: GET https://rp.j5719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80008e-4b66-48be-b60f-ca94f871de32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80008e-4b66-48be-b60f-ca94f871de32\",\n \"name\": \"1a80008e-4b66-48be-b60f-ca94f871de32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:56:32.579892703Z\",\n \"endTime\": \"2026-09-17T22:15:39.575240678Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691087415840768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T20:59:18Z","started_at":"2026-09-17T20:59:18Z","completed_at":"2026-09-17T21:18:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:02.929473189Z","timing_metadata_checked_at":"2026-09-18T15:41:02.929473189Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:02.933177402Z","region_metadata_checked_at":"2026-09-19T03:30:39.453570283Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fa760241ff8512435538e40e151fdb36c2a6921889dad5d6d8879f1546bf14da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691087415840768","occurred_at":"2026-09-17T20:59:18Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691038216654848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6964,"pr_state":"closed","pr_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","final_merged_sha":"af707ce72df7d034eaee96beafbefdfe37fc0800","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T20:59:07Z","started_at":"2026-09-17T20:59:07Z","completed_at":"2026-09-17T22:00:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:59.063543475Z","timing_metadata_checked_at":"2026-09-18T15:40:59.063543475Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:59.066775586Z","region_metadata_checked_at":"2026-09-18T15:40:59.066775586Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8df46c79a83bc46523d87e7f8f95a47693553d8495fc8f5a887b5f90bd4e4eec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691038216654848","occurred_at":"2026-09-17T20:59:07Z","failed_at":"provision","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-17T21:21:29.249Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T21:21:29.969Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-17T21:21:30.723Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=63b12b492d5d28b0b0b18408fa469c00c31993867db99322c5926f55f6db447a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6654848-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F63b12b492d5d28b0b0b18408fa469c00c31993867db99322c5926f55f6db447a\ntime=2026-09-17T21:51:28.943Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100689397040025600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"344450021ef1cd34320edd7790bea47a65b8b30e","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T20:52:35Z","started_at":"2026-09-17T20:52:35Z","completed_at":"2026-09-17T21:55:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:53.730743894Z","timing_metadata_checked_at":"2026-09-18T15:40:53.730743894Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:53.733827305Z","region_metadata_checked_at":"2026-09-18T15:40:53.733827305Z"},"failed_at":["alert","provision"],"failed_test_count":2,"occurrences":[{"row_id":"85b667d83a0d57d9545a318dd8536bfe9a4ca4230cedb5622eb17d16b7ca3fdc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100689397040025600","occurred_at":"2026-09-17T20:52:35Z","failed_at":"provision","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-17T21:19:38.472Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T21:19:39.145Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-17T21:19:39.871Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=f518dca3cc0829098231fe8a127f84b76b966486264714a9e41a0d1fd09997db portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0025600-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff518dca3cc0829098231fe8a127f84b76b966486264714a9e41a0d1fd09997db\ntime=2026-09-17T21:49:38.148Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}},{"row_id":"dddb1cae9b6d93f5f18b1ef993dd86878864285f6e14981b68132ec3185d5d39","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100689397040025600","occurred_at":"2026-09-17T20:52:35Z","failed_at":"alert","signature_id":"c8d2c94c46ea41bb6616321fc23a840d9b2ec094985319abefcb6af50ae87597","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T21:40:23Z\n Severity: Sev3\n Labels: alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j0025600-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685757428535296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6998,"pr_state":"closed","pr_sha":"c947fb99750b0318d83264c65a0eebdbade0c5e1","final_merged_sha":"c947fb99750b0318d83264c65a0eebdbade0c5e1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T20:38:07Z","started_at":"2026-09-17T20:38:07Z","completed_at":"2026-09-17T22:46:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:17.968939982Z","timing_metadata_checked_at":"2026-09-18T15:41:17.968939982Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:17.974521202Z","region_metadata_checked_at":"2026-09-18T15:41:17.974521202Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"a06429f389bbc03ab02243f5c98df0573557f1685314f385cfe8b97a1eaed74e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685757428535296","occurred_at":"2026-09-17T20:38:07Z","failed_at":"e2e","signature_id":"f8f97e2decd1a3d48fb001cc1c33150efca6eca10d42b34bfc0118d3f5dd19c8","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011081a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-wrb88wtb5d8f\" to finish creating: GET https://rp.j8535296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4b63f0-45c8-411e-81ab-a9132fbbad49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4b63f0-45c8-411e-81ab-a9132fbbad49\",\n \"name\": \"ff4b63f0-45c8-411e-81ab-a9132fbbad49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:37:11.907762823Z\",\n \"endTime\": \"2026-09-17T22:06:19.7850947Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0011081a0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-wrb88wtb5d8f\" to finish creating: GET https://rp.j8535296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4b63f0-45c8-411e-81ab-a9132fbbad49\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff4b63f0-45c8-411e-81ab-a9132fbbad49\",\n \"name\": \"ff4b63f0-45c8-411e-81ab-a9132fbbad49\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:37:11.907762823Z\",\n \"endTime\": \"2026-09-17T22:06:19.7850947Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a2acd803de3deda29406c758dc2f443d976a3b2ef31a2a9c690ae744bbe11105","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685757428535296","occurred_at":"2026-09-17T20:38:07Z","failed_at":"alert","signature_id":"927ffe3fc5118bce586a8a07b93ed80685d7fe4d247724a208069edabb849e6f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T21:50:34Z\n Ended: 2026-09-17T22:18:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8535296-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st0v24nqnshekebv9kf1to2rdotufnh-v20261001\", pod=\"router-b886fb777-8chsd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2st0v24nqnshekebv9kf1to2rdotufnh-v20261001/router-b886fb777-8chsd has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"ea503ec41ff47705c9a97cd58f7750bc3a983568","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T20:37:43Z","started_at":"2026-09-17T20:37:43Z","completed_at":"2026-09-17T22:48:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.064841494Z","timing_metadata_checked_at":"2026-09-18T15:41:21.064841494Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:21.068400107Z","region_metadata_checked_at":"2026-09-18T15:41:21.068400107Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"033003cffc6b2eb73a19c197bd13b950da70153d725b360ac68bcdd42dbad31c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","failed_at":"e2e","signature_id":"68698c4e2d81e96a811614a8eb209ad22dc4f07b9b07bee4f975ea46939d038d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-whlxpq8kpbjp/cluster-candidate-4-20-fzx8w5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa2260\u003e: \n failed to create HCP cluster cluster-candidate-4-20-fzx8w5: failed waiting for cluster=\"cluster-candidate-4-20-fzx8w5\" in resourcegroup=\"rg-candidate-4-20-whlxpq8kpbjp\" to finish creating: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/15ce6538-14f5-43b6-95c3-8e840903f713\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/15ce6538-14f5-43b6-95c3-8e840903f713\",\n \"name\": \"15ce6538-14f5-43b6-95c3-8e840903f713\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:28:54.798993309Z\",\n \"endTime\": \"2026-09-17T21:58:02.391238314Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 24m21s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-whlxpq8kpbjp/cluster-candidate-4-20-fzx8w5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa2260\u003e: \n failed to create HCP cluster cluster-candidate-4-20-fzx8w5: failed waiting for cluster=\"cluster-candidate-4-20-fzx8w5\" in resourcegroup=\"rg-candidate-4-20-whlxpq8kpbjp\" to finish creating: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/15ce6538-14f5-43b6-95c3-8e840903f713\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/15ce6538-14f5-43b6-95c3-8e840903f713\",\n \"name\": \"15ce6538-14f5-43b6-95c3-8e840903f713\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:28:54.798993309Z\",\n \"endTime\": \"2026-09-17T21:58:02.391238314Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 24m21s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2f0350e60eee14f80f67d259c37cb4e0d333c2f0013e324bea63da3fb95b8fc3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","failed_at":"e2e","signature_id":"fa245128857c23599fcf8f3484b06ab6ca5868cca2b19c36072deb31d3760300","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0004ec1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hdsrgp87vhp6\" to finish deleting: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\",\n \"name\": \"140fc1a6-e41e-4f25-9908-25d174a04842\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:51:08.271574522Z\",\n \"endTime\": \"2026-09-17T22:15:12.38485138Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st0r3e7gsp00p1v6je2veqpisj6tv8b still exists (deletion dispatched at 2026-09-17T21:51:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0004ec1e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hdsrgp87vhp6\" to finish deleting: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\",\n \"name\": \"140fc1a6-e41e-4f25-9908-25d174a04842\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:51:08.271574522Z\",\n \"endTime\": \"2026-09-17T22:15:12.38485138Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st0r3e7gsp00p1v6je2veqpisj6tv8b still exists (deletion dispatched at 2026-09-17T21:51:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bbca654fa7f218d08a86486c1ba25802dc22276e667ea34270423a190d18c163","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","failed_at":"alert","signature_id":"a25eab3d2aea0534a58060ed60f2bfb99bd3c649f3ff0b27d214e0eebb1f8f68","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T22:35:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6073984-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.6:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6073984/providers/microsoft.monitor/accounts/hcps-j6073984\", namespace=\"ocm-arohcpci01-2st0r3e7gsp00p1v6je2veqpisj6tv8b-a8q3g3x9d3e6x7g\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-7mb8q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2st0r3e7gsp00p1v6je2veqpisj6tv8b-a8q3g3x9d3e6x7g/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T22:35:22Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6073984-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.6:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6073984/providers/microsoft.monitor/accounts/hcps-j6073984\", namespace=\"ocm-arohcpci01-2st0r3e7gsp00p1v6je2veqpisj6tv8b-a8q3g3x9d3e6x7g\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-7mb8q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2st0r3e7gsp00p1v6je2veqpisj6tv8b-a8q3g3x9d3e6x7g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680830069248000","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T20:18:28Z","started_at":"2026-09-17T20:18:28Z","completed_at":"2026-09-17T22:20:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:32.991032162Z","timing_metadata_checked_at":"2026-09-18T15:41:32.991032162Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:32.994019972Z","region_metadata_checked_at":"2026-09-18T15:41:32.994019972Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"8bccd883802213d6f2eaa418fd8acd9b2cd4c3fa","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T20:17:48Z","started_at":"2026-09-17T20:17:48Z","completed_at":"2026-09-17T22:22:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:19.975401848Z","timing_metadata_checked_at":"2026-09-18T15:41:19.975401848Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:19.981151069Z","region_metadata_checked_at":"2026-09-18T15:41:19.981151069Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"15d200bcfc8c09a46c8160ba5aa685dbaf39de77a17ad41c5bcec888f9f305e8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","failed_at":"alert","signature_id":"f3ebbdf3ff2dce8e5f0f8347ed14b1024c81d64efecc73041c83a810441bae40","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T21:40:09Z\n Ended: 2026-09-17T21:58:09Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j1728768-svc\", component=\"backend\", name=\"externalauthclusterservicecreate\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue externalauthclusterservicecreate has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired"}},{"row_id":"7c74f914efeec56bfc3476bc293d32c14eaa3c66ae11b8a793b9975d68a4e7f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","failed_at":"e2e","signature_id":"bf84ee4449e687d0ad4cd4859ef7515e9901f5ded80364d34ebb748aaa5b41df","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002990240\u003e: \n failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc002990240\u003e: \n failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4f70c086dc5495ea728fc39aab23a54b6cce4072ba8ccfc36ee6980300ba5299","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded"}},{"row_id":"9f75256cdcfced3459d0be2cc57fd5188a4ad3d57e9611d3284d318f40886fe8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","failed_at":"alert","signature_id":"d9ede6c71c00bb7dbee4ddd3fb9aecba02c72aceca980c3acfa9826e45ff50ea","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T21:49:48Z\n Ended: 2026-09-17T22:07:50Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1728768-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters has exceeded 250ms over the past 30 minutes (cluster ci01-j1728768-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["4f70c086dc5495ea728fc39aab23a54b6cce4072ba8ccfc36ee6980300ba5299","98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680100679782400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T20:15:39Z","started_at":"2026-09-17T20:15:39Z","completed_at":"2026-09-17T20:16:18Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:56.582660852Z","timing_metadata_checked_at":"2026-09-18T15:40:56.582660852Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:56.586197365Z","region_metadata_checked_at":"2026-09-19T03:30:38.385355554Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6ff1286c629d004955c3a22490637ee316547689ba04a1c86c69b11b70812451","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680100679782400","occurred_at":"2026-09-17T20:15:39Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100678231362375680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T20:03:18Z","started_at":"2026-09-17T20:03:18Z","completed_at":"2026-09-17T22:15:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:32.85457567Z","timing_metadata_checked_at":"2026-09-18T15:41:32.85457567Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:32.857927582Z","region_metadata_checked_at":"2026-09-18T15:41:32.857927582Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"9ffe1dcd1e495ec2764fe25c97274a3d6d2bda9236952db2ae5cf38d5ecc49fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100678231362375680","occurred_at":"2026-09-17T20:03:18Z","failed_at":"e2e","signature_id":"14578415f39945108c1077bc9568d80eb9c57e111ae4ccc29677f2723df82376","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8c920\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-z9b9d4ng6qd2\" to finish creating: GET https://rp.j2375680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\",\n \"name\": \"ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:14:44.291403118Z\",\n \"endTime\": \"2026-09-17T21:33:45.719372752Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c8c920\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-z9b9d4ng6qd2\" to finish creating: GET https://rp.j2375680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\",\n \"name\": \"ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T21:14:44.291403118Z\",\n \"endTime\": \"2026-09-17T21:33:45.719372752Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100677863932956672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5562,"pr_state":"open","pr_sha":"e0440b68b80a986602a8d6b89cd125f3b510e8b8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T20:01:43Z","started_at":"2026-09-17T20:01:43Z","completed_at":"2026-09-17T20:07:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:02.084969778Z","timing_metadata_checked_at":"2026-09-18T15:40:02.084969778Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:02.090301197Z","region_metadata_checked_at":"2026-09-19T03:30:32.33692925Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e141208df11e9d4e4d17473a148b2696ec448b1408840b87ab300890fef10cf1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100677863932956672","occurred_at":"2026-09-17T20:01:43Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100636618372157440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6968,"pr_state":"open","pr_sha":"6e7413f9261ae3f89f1d8b9b4817769e713e32da","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T17:22:52Z","started_at":"2026-09-17T17:22:52Z","completed_at":"2026-09-17T17:26:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:01.04122309Z","timing_metadata_checked_at":"2026-09-18T15:41:01.04122309Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:01.044408101Z","region_metadata_checked_at":"2026-09-19T03:30:38.943861714Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8f14311d8e2e977cfb0184ec373d90b903f7cbfb6db886879ab0d0c30d8ba083","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100636618372157440","occurred_at":"2026-09-17T17:22:52Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100636207791738880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T17:21:14Z","started_at":"2026-09-17T17:21:14Z","completed_at":"2026-09-17T17:21:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:32.724919203Z","timing_metadata_checked_at":"2026-09-18T15:41:32.724919203Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:32.728383915Z","region_metadata_checked_at":"2026-09-19T03:30:50.567257589Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ddf385f766e072f04ea6a30dbf7571863cf74af5fee44b1882439864f2eb812e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100636207791738880","occurred_at":"2026-09-17T17:21:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6858/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100635900047265792","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6858,"pr_state":"open","pr_sha":"150b9bf5d82a4bcc1f959e465ffe6769e739370b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T17:20:01Z","started_at":"2026-09-17T17:20:01Z","completed_at":"2026-09-17T17:26:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:29.247799519Z","timing_metadata_checked_at":"2026-09-18T15:40:29.247799519Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:29.256240149Z","region_metadata_checked_at":"2026-09-19T03:30:35.447492921Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f63372099f15f3fe5aebb0ef76841199bdfe33945abc030a135cad8b87bbf6d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6858/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100635900047265792","occurred_at":"2026-09-17T17:20:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100635512048979968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T17:18:28Z","started_at":"2026-09-17T17:18:28Z","completed_at":"2026-09-17T17:18:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:32.597772744Z","timing_metadata_checked_at":"2026-09-18T15:41:32.597772744Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:32.600873255Z","region_metadata_checked_at":"2026-09-19T03:30:50.477226759Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"401910b878f0e05714f5f0584bdee9c1965b4a25d31e8cad8e8b269c485fbea2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100635512048979968","occurred_at":"2026-09-17T17:18:28Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100634780142931968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T17:15:26Z","started_at":"2026-09-17T17:15:26Z","completed_at":"2026-09-17T17:16:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:32.461644053Z","timing_metadata_checked_at":"2026-09-18T15:41:32.461644053Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:32.464708964Z","region_metadata_checked_at":"2026-09-19T03:30:48.802446429Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0b115aa9f89e3b275d923471ec2e7081fd89ac1fd216957ef04773ce0149691b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100634780142931968","occurred_at":"2026-09-17T17:15:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100633977030184960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T17:12:22Z","started_at":"2026-09-17T17:12:22Z","completed_at":"2026-09-17T17:12:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:32.317100132Z","timing_metadata_checked_at":"2026-09-18T15:41:32.317100132Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:32.320335144Z","region_metadata_checked_at":"2026-09-19T03:30:48.714632208Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"159732b25c85ae05ce6644e91e0d968adcc9398636fc29a8e3c78fcd59874a2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100633977030184960","occurred_at":"2026-09-17T17:12:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100633742971244544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6969,"pr_state":"open","pr_sha":"094e2b2bacb64504d29203e7b04241f70f4afc54","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T17:11:22Z","started_at":"2026-09-17T17:11:22Z","completed_at":"2026-09-17T17:15:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:01.658866514Z","timing_metadata_checked_at":"2026-09-18T15:41:01.658866514Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:01.662032325Z","region_metadata_checked_at":"2026-09-19T03:30:39.046674692Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e0da8ccf247f61469aba5a8728fc839d1cd8fa38f05b24a3d0a85592056e1eb2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100633742971244544","occurred_at":"2026-09-17T17:11:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100633528617144320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7001,"pr_state":"open","pr_sha":"7abdd6eae77d77653cbfd112bce953b5d146232a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T17:10:35Z","started_at":"2026-09-17T17:10:35Z","completed_at":"2026-09-17T17:14:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:21.538397909Z","timing_metadata_checked_at":"2026-09-18T15:41:21.538397909Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:21.54143242Z","region_metadata_checked_at":"2026-09-19T03:30:41.565667021Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"fdbbc9e7c99a0afb9f866f1a0b4a450eedc098414dda9361d413761855ff666a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7001/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100633528617144320","occurred_at":"2026-09-17T17:10:35Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100633290145796096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6807,"pr_state":"open","pr_sha":"257db47c10ddb67d7539ad624828ade0c814c1ee","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T17:09:38Z","started_at":"2026-09-17T17:09:38Z","completed_at":"2026-09-17T17:14:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:24.824958007Z","timing_metadata_checked_at":"2026-09-18T15:40:24.824958007Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:24.829343822Z","region_metadata_checked_at":"2026-09-19T03:30:35.256995818Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a55ca8483bf1679fe458f48dbbc6d1b6ecccda975a61181cdc6c28273120c94e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100633290145796096","occurred_at":"2026-09-17T17:09:38Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100632986532712448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T17:08:26Z","started_at":"2026-09-17T17:08:26Z","completed_at":"2026-09-17T17:08:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:32.195973595Z","timing_metadata_checked_at":"2026-09-18T15:41:32.195973595Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:32.200386111Z","region_metadata_checked_at":"2026-09-19T03:30:48.624304277Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ece169d13c8b0dcd225771d446f73edb97f3e6b2b35334f9e2765fe9e4af5cec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100632986532712448","occurred_at":"2026-09-17T17:08:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100631986799382528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T17:04:28Z","started_at":"2026-09-17T17:04:28Z","completed_at":"2026-09-17T17:04:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:32.066148527Z","timing_metadata_checked_at":"2026-09-18T15:41:32.066148527Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:32.069234038Z","region_metadata_checked_at":"2026-09-19T03:30:48.528839527Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8850920025adedfe599aca2baf490f5ba2f44c996875e830ae267d8ac8e3007c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100631986799382528","occurred_at":"2026-09-17T17:04:28Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100631231749165056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T17:01:24Z","started_at":"2026-09-17T17:01:24Z","completed_at":"2026-09-17T17:01:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:31.940459873Z","timing_metadata_checked_at":"2026-09-18T15:41:31.940459873Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:31.943586085Z","region_metadata_checked_at":"2026-09-19T03:30:48.437573693Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"aad0a06e4a35e310066585cf911c94e74e4cc4485c78c9f4847ed5d1d0b3a08e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100631231749165056","occurred_at":"2026-09-17T17:01:24Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100631067403751424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"ea503ec41ff47705c9a97cd58f7750bc3a983568","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T17:00:27Z","started_at":"2026-09-17T17:00:27Z","completed_at":"2026-09-17T17:04:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.926518193Z","timing_metadata_checked_at":"2026-09-18T15:41:20.926518193Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:20.929470004Z","region_metadata_checked_at":"2026-09-19T03:30:41.465825655Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4eada4ada4cbb6b8e161927b142577a68544a461218449be423ba38ec20595ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100631067403751424","occurred_at":"2026-09-17T17:00:27Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100630346600026112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:57:57Z","started_at":"2026-09-17T16:57:57Z","completed_at":"2026-09-17T16:58:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:31.809443501Z","timing_metadata_checked_at":"2026-09-18T15:41:31.809443501Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:31.813126214Z","region_metadata_checked_at":"2026-09-19T03:30:48.329142296Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"88cc0e17a36b0c307b17e674ab0e94cfabecb56baf66dea6b183d4f01875e22c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100630346600026112","occurred_at":"2026-09-17T16:57:57Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100629631555080192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:54:58Z","started_at":"2026-09-17T16:54:58Z","completed_at":"2026-09-17T16:55:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:34.850717169Z","timing_metadata_checked_at":"2026-09-18T15:40:34.850717169Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:34.854582583Z","region_metadata_checked_at":"2026-09-19T03:30:36.074742034Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8deaf7e0df8bdddd76d3198ff2f01df7fa2841a8beb3be21e20a7a1d80c0f81f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100629631555080192","occurred_at":"2026-09-17T16:54:58Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100629601368674304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:54:53Z","started_at":"2026-09-17T16:54:53Z","completed_at":"2026-09-17T16:55:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:31.690813673Z","timing_metadata_checked_at":"2026-09-18T15:41:31.690813673Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:31.693842584Z","region_metadata_checked_at":"2026-09-19T03:30:47.95574323Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9d2c193b8bbd5e67127737790f3948ba6fabb71b24ed816bce83d86a5c144f0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100629601368674304","occurred_at":"2026-09-17T16:54:53Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100628533251411968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:50:44Z","started_at":"2026-09-17T16:50:44Z","completed_at":"2026-09-17T16:51:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:31.552866176Z","timing_metadata_checked_at":"2026-09-18T15:41:31.552866176Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:31.558382795Z","region_metadata_checked_at":"2026-09-19T03:30:47.869559314Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"dee28a9b825cae4489b9204474e751fcc2e99afa4c9863c63995f1fbabb09c45","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100628533251411968","occurred_at":"2026-09-17T16:50:44Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100627397169647616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:46:13Z","started_at":"2026-09-17T16:46:13Z","completed_at":"2026-09-17T16:46:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:31.421802203Z","timing_metadata_checked_at":"2026-09-18T15:41:31.421802203Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:31.425268915Z","region_metadata_checked_at":"2026-09-19T03:30:47.767164239Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d76fc61fcc57bd6383650f7a0f0b2670e9de2abdb288a6cf511c951352651c4f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100627397169647616","occurred_at":"2026-09-17T16:46:13Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100626368973770752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:42:08Z","started_at":"2026-09-17T16:42:08Z","completed_at":"2026-09-17T16:42:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:31.288874023Z","timing_metadata_checked_at":"2026-09-18T15:41:31.288874023Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:31.292184535Z","region_metadata_checked_at":"2026-09-19T03:30:47.462718925Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0e23f6ec3bd0102c3cd894d3701ec61a9acdaeb2836ea217e5a69507ffd75263","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100626368973770752","occurred_at":"2026-09-17T16:42:08Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100625584206909440","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:39:01Z","started_at":"2026-09-17T16:39:01Z","completed_at":"2026-09-17T16:39:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:31.193247778Z","timing_metadata_checked_at":"2026-09-18T15:41:31.193247778Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:31.19655599Z","region_metadata_checked_at":"2026-09-19T03:30:47.370662288Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2551dccf4431b2458a9d701e95fa6f91638f99371c07d22d3b82cfcfa22286a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100625584206909440","occurred_at":"2026-09-17T16:39:01Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100624815797833728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:35:58Z","started_at":"2026-09-17T16:35:58Z","completed_at":"2026-09-17T16:36:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:31.067596625Z","timing_metadata_checked_at":"2026-09-18T15:41:31.067596625Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:31.070904537Z","region_metadata_checked_at":"2026-09-19T03:30:47.266175106Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5aa25c3fa7c0dddb34e1f2296de0c0cf3910e826663edb98fb1efe79e8e51b1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100624815797833728","occurred_at":"2026-09-17T16:35:58Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100624045081890816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:32:54Z","started_at":"2026-09-17T16:32:54Z","completed_at":"2026-09-17T16:33:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:30.974884091Z","timing_metadata_checked_at":"2026-09-18T15:41:30.974884091Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:30.978017502Z","region_metadata_checked_at":"2026-09-19T03:30:47.070084088Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6983676d504d5b60165dd034b966106f83aa13a67c27bf28218ab25824eda7f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100624045081890816","occurred_at":"2026-09-17T16:32:54Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100623183672512512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:29:17Z","started_at":"2026-09-17T16:29:17Z","completed_at":"2026-09-17T16:29:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:30.847947733Z","timing_metadata_checked_at":"2026-09-18T15:41:30.847947733Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:30.851995148Z","region_metadata_checked_at":"2026-09-19T03:30:46.982008265Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1518c81ab713a54e4011b6dc339e8513612426c775ec6bb6f46022642b6184d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100623183672512512","occurred_at":"2026-09-17T16:29:17Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100622375014895616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:26:16Z","started_at":"2026-09-17T16:26:16Z","completed_at":"2026-09-17T16:26:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:02.840189768Z","timing_metadata_checked_at":"2026-09-18T15:41:02.840189768Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:02.843351179Z","region_metadata_checked_at":"2026-09-19T03:30:39.350712306Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d564e431af02b448841ba2da2073d26a3de9362cb3da782d51794b0b7d0d3256","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100622375014895616","occurred_at":"2026-09-17T16:26:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100622183377145856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"8bccd883802213d6f2eaa418fd8acd9b2cd4c3fa","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T16:25:30Z","started_at":"2026-09-17T16:25:30Z","completed_at":"2026-09-17T16:29:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:19.811343754Z","timing_metadata_checked_at":"2026-09-18T15:41:19.811343754Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:19.814613166Z","region_metadata_checked_at":"2026-09-19T03:30:41.182738818Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"cc0a7981e1550c744908ff4a4e90505361d03c51bdf07c2fb1b3e9a8d663d1ce","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100622183377145856","occurred_at":"2026-09-17T16:25:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100622127337050112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:25:17Z","started_at":"2026-09-17T16:25:17Z","completed_at":"2026-09-17T16:25:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:30.688565958Z","timing_metadata_checked_at":"2026-09-18T15:41:30.688565958Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:30.691992371Z","region_metadata_checked_at":"2026-09-19T03:30:46.885845213Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9af6110f2af3bb395b22910c9b61b061d00934bbe85cfdfbb4a302b79a7d2481","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100622127337050112","occurred_at":"2026-09-17T16:25:17Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100620327796084736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6930,"pr_state":"closed","pr_sha":"8457819ce36966a8fbb8ae4bb521ffc1fdb54e00","final_merged_sha":"8457819ce36966a8fbb8ae4bb521ffc1fdb54e00","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:18:08Z","started_at":"2026-09-17T16:18:08Z","completed_at":"2026-09-17T16:21:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:30.584239782Z","timing_metadata_checked_at":"2026-09-18T15:41:30.584239782Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:30.588454897Z","region_metadata_checked_at":"2026-09-19T03:30:46.725198824Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"205d754011c454192cb9fa84d97854995da0ac5ed18b09d3cbaca1249965c745","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100620327796084736","occurred_at":"2026-09-17T16:18:08Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100619360547639296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:14:17Z","started_at":"2026-09-17T16:14:17Z","completed_at":"2026-09-17T16:14:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:30.475078288Z","timing_metadata_checked_at":"2026-09-18T15:41:30.475078288Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:30.478901402Z","region_metadata_checked_at":"2026-09-19T03:30:46.63124498Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"850352510c42bebe182e9eba41df77293bc3653c2dabfec2fc23e83fea0191a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100619360547639296","occurred_at":"2026-09-17T16:14:17Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100619267484422144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:13:55Z","started_at":"2026-09-17T16:13:55Z","completed_at":"2026-09-17T16:14:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:34.761768549Z","timing_metadata_checked_at":"2026-09-18T15:40:34.761768549Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:34.76478496Z","region_metadata_checked_at":"2026-09-19T03:30:35.955936895Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"328c12121333eac8648bed4c34283fc64766ad7d7f97ac8b88138a606b45ab83","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100619267484422144","occurred_at":"2026-09-17T16:13:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100618376668778496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:10:19Z","started_at":"2026-09-17T16:10:19Z","completed_at":"2026-09-17T16:11:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:29.352039438Z","timing_metadata_checked_at":"2026-09-18T15:41:29.352039438Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:29.354993048Z","region_metadata_checked_at":"2026-09-19T03:30:46.539995246Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"29134cedba86b1db23876fdcbd70657935653e1f09fd90794f8a51a7da75fe78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100618376668778496","occurred_at":"2026-09-17T16:10:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100617585681764352","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:07:11Z","started_at":"2026-09-17T16:07:11Z","completed_at":"2026-09-17T16:07:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:29.222930172Z","timing_metadata_checked_at":"2026-09-18T15:41:29.222930172Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:29.226503385Z","region_metadata_checked_at":"2026-09-19T03:30:46.453303528Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ea0a1b09f64abbbb5a1c78c7186b6f1fe15d02dcb110410542c7d275cdb53c16","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100617585681764352","occurred_at":"2026-09-17T16:07:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100616698091540480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"344450021ef1cd34320edd7790bea47a65b8b30e","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T16:03:42Z","started_at":"2026-09-17T16:03:42Z","completed_at":"2026-09-17T16:04:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:53.608083553Z","timing_metadata_checked_at":"2026-09-18T15:40:53.608083553Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:53.611169364Z","region_metadata_checked_at":"2026-09-19T03:30:38.189355131Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f7e8a28597757756ba34b162c263241b35a186d00f283036aa1134fe4d864ba5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100616698091540480","occurred_at":"2026-09-17T16:03:42Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100616616722042880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:03:21Z","started_at":"2026-09-17T16:03:21Z","completed_at":"2026-09-17T16:03:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:29.101637535Z","timing_metadata_checked_at":"2026-09-18T15:41:29.101637535Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:29.104728746Z","region_metadata_checked_at":"2026-09-19T03:30:46.365262606Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b1a1e7d0dc6d985a888da5712f94683a7c0ec6c1681ba498c1043d5e2b28ac8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100616616722042880","occurred_at":"2026-09-17T16:03:21Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100616108892491776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"344450021ef1cd34320edd7790bea47a65b8b30e","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T16:01:22Z","started_at":"2026-09-17T16:01:22Z","completed_at":"2026-09-17T16:01:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:53.516455823Z","timing_metadata_checked_at":"2026-09-18T15:40:53.516455823Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:53.520320737Z","region_metadata_checked_at":"2026-09-19T03:30:38.095178784Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"741c658410465bb8e488c1c63eceb59bde3b4d462e67633dc36d6721b28dce22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100616108892491776","occurred_at":"2026-09-17T16:01:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615750157864960","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"ea503ec41ff47705c9a97cd58f7750bc3a983568","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:59:55Z","started_at":"2026-09-17T15:59:55Z","completed_at":"2026-09-17T16:00:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.806225257Z","timing_metadata_checked_at":"2026-09-18T15:41:20.806225257Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:20.810635773Z","region_metadata_checked_at":"2026-09-19T03:30:41.365499188Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b5f79c6c822d8cad53b3bbeeea6a58eaa08a9cc6791266d0e73a131ec24fa62b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615750157864960","occurred_at":"2026-09-17T15:59:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615710781739008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"3f0521ff633af0368f049bfead3093e31565b50f","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:59:47Z","started_at":"2026-09-17T15:59:47Z","completed_at":"2026-09-17T16:05:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:35.655373663Z","timing_metadata_checked_at":"2026-09-18T15:40:35.655373663Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:35.66017978Z","region_metadata_checked_at":"2026-09-19T03:30:36.270076154Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e7ecd972629007fc8e7691f89196ae62f3f7e03efe2071b26c71e8eedc5d55db","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615710781739008","occurred_at":"2026-09-17T15:59:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615692955947008","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6988,"pr_state":"closed","pr_sha":"53b8fd9bee7c04069b9537ac28ccedffde355125","final_merged_sha":"53b8fd9bee7c04069b9537ac28ccedffde355125","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:59:42Z","started_at":"2026-09-17T15:59:42Z","completed_at":"2026-09-17T16:05:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.139369701Z","timing_metadata_checked_at":"2026-09-18T15:41:12.139369701Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:12.145271722Z","region_metadata_checked_at":"2026-09-19T03:30:40.305894806Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6898549b53a5d419588b42d3cb2e87281ef7007fe8a12bd6554f94a9dbbe3ef8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615692955947008","occurred_at":"2026-09-17T15:59:42Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615664686338048","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:59:36Z","started_at":"2026-09-17T15:59:36Z","completed_at":"2026-09-17T16:03:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:02.714637616Z","timing_metadata_checked_at":"2026-09-18T15:41:02.714637616Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:02.717925427Z","region_metadata_checked_at":"2026-09-19T03:30:39.245434221Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"dafab50ae5c70f9722f436f04a16977c9f8147cbdd76f636884637852c568f1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615664686338048","occurred_at":"2026-09-17T15:59:36Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615647078649856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:59:32Z","started_at":"2026-09-17T15:59:32Z","completed_at":"2026-09-17T16:05:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:46.719180776Z","timing_metadata_checked_at":"2026-09-18T15:40:46.719180776Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:46.724921696Z","region_metadata_checked_at":"2026-09-19T03:30:37.778620117Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"311393247616553850434794d839cc8141b642be9f530bbf191b8865b2a3c362","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615647078649856","occurred_at":"2026-09-17T15:59:32Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6930/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615577109270528","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6930,"pr_state":"closed","pr_sha":"8457819ce36966a8fbb8ae4bb521ffc1fdb54e00","final_merged_sha":"8457819ce36966a8fbb8ae4bb521ffc1fdb54e00","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:59:15Z","started_at":"2026-09-17T15:59:15Z","completed_at":"2026-09-17T16:02:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:42.621200631Z","timing_metadata_checked_at":"2026-09-18T15:40:42.621200631Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:42.624278542Z","region_metadata_checked_at":"2026-09-19T03:30:36.630836584Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"cd0004062a430eaa6ee04feed61bc0d81eaa2c1127f78238bed0ba7423c8a555","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6930/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615577109270528","occurred_at":"2026-09-17T15:59:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615535338196992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:59:05Z","started_at":"2026-09-17T15:59:05Z","completed_at":"2026-09-17T15:59:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:28.987004421Z","timing_metadata_checked_at":"2026-09-18T15:41:28.987004421Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:28.990471734Z","region_metadata_checked_at":"2026-09-19T03:30:46.272906267Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"09baa52579bcd067337c858dbd0c3c19b3a7c2b640bc4d60e71bc7ea00360a1c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100615535338196992","occurred_at":"2026-09-17T15:59:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100614649383751680","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:55:21Z","started_at":"2026-09-17T15:55:21Z","completed_at":"2026-09-17T15:55:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:28.817838011Z","timing_metadata_checked_at":"2026-09-18T15:41:28.817838011Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:28.821012123Z","region_metadata_checked_at":"2026-09-19T03:30:46.181323032Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"858842b9c2be0b80d82f5a662916ae9de67fd0a4cac3a400cf1c9ee0997a78b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100614649383751680","occurred_at":"2026-09-17T15:55:21Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100614240082595840","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6945,"pr_state":"closed","pr_sha":"344450021ef1cd34320edd7790bea47a65b8b30e","final_merged_sha":"ad72b9c83f427d87b930e00e09a82f50fdeb7aa2","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:53:56Z","started_at":"2026-09-17T15:53:56Z","completed_at":"2026-09-17T15:59:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:52.573397932Z","timing_metadata_checked_at":"2026-09-18T15:40:52.573397932Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:52.578690651Z","region_metadata_checked_at":"2026-09-19T03:30:37.998519828Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a92f0968284650cce77d6cf6898bac315a7eb3b6ea7e4623bbc1eacfd868875a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100614240082595840","occurred_at":"2026-09-17T15:53:56Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100614100391301120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"8bccd883802213d6f2eaa418fd8acd9b2cd4c3fa","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:53:23Z","started_at":"2026-09-17T15:53:23Z","completed_at":"2026-09-17T15:56:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:19.715085806Z","timing_metadata_checked_at":"2026-09-18T15:41:19.715085806Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:19.718682719Z","region_metadata_checked_at":"2026-09-19T03:30:41.095143497Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e9480fb4f4aa15209839a9135e416478a652b39c64aa12d67ed497b0c8bdc262","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100614100391301120","occurred_at":"2026-09-17T15:53:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100613648123695104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:51:23Z","started_at":"2026-09-17T15:51:23Z","completed_at":"2026-09-17T15:51:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:28.699527784Z","timing_metadata_checked_at":"2026-09-18T15:41:28.699527784Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:28.702933097Z","region_metadata_checked_at":"2026-09-19T03:30:46.054338367Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"efd2807985a8a4aadb8ee332f15aaa15a76a3f4b4dfb3d66ce2f2c756dbc5526","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100613648123695104","occurred_at":"2026-09-17T15:51:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100613310561914880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7000,"pr_state":"closed","pr_sha":"ea503ec41ff47705c9a97cd58f7750bc3a983568","final_merged_sha":"e253ee7d7a59b02821f6339251ae15272721a63a","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:50:15Z","started_at":"2026-09-17T15:50:15Z","completed_at":"2026-09-17T15:53:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:20.717744337Z","timing_metadata_checked_at":"2026-09-18T15:41:20.717744337Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:20.721149849Z","region_metadata_checked_at":"2026-09-19T03:30:41.274741755Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8853e2d50f2a4d49676776148f4fa7f72ecb0bbefcbeb4386c3719c2484f2805","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100613310561914880","occurred_at":"2026-09-17T15:50:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100612675183579136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:47:26Z","started_at":"2026-09-17T15:47:26Z","completed_at":"2026-09-17T15:48:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:28.561467986Z","timing_metadata_checked_at":"2026-09-18T15:41:28.561467986Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:28.5651322Z","region_metadata_checked_at":"2026-09-19T03:30:45.900380002Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8c6b191183872b5cbf5f65b2175e4967dcac21c68a16a92308e9ad2e75a19864","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100612675183579136","occurred_at":"2026-09-17T15:47:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100611721499512832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:43:47Z","started_at":"2026-09-17T15:43:47Z","completed_at":"2026-09-17T15:44:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:28.423593989Z","timing_metadata_checked_at":"2026-09-18T15:41:28.423593989Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:28.4266031Z","region_metadata_checked_at":"2026-09-19T03:30:45.80151134Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"1a1d993804dfe5585f1dda2746cea8e7b385c0aba460ceb961ff1f86563b8f5f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100611721499512832","occurred_at":"2026-09-17T15:43:47Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100610728342851584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:39:53Z","started_at":"2026-09-17T15:39:53Z","completed_at":"2026-09-17T15:40:19Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:28.30744627Z","timing_metadata_checked_at":"2026-09-18T15:41:28.30744627Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:28.310467381Z","region_metadata_checked_at":"2026-09-19T03:30:45.703889283Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b208f225f2fd6ab35e515306d78ebd1a91f8421372faef3fcfb8ae4785e9b6a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100610728342851584","occurred_at":"2026-09-17T15:39:53Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100610270391963648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"closed","pr_sha":"728300dde5778eae2cc566e635f9026903fc274b","final_merged_sha":"54049e5d9f45d43158a9be93d63e7e1835e4a01f","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:38:04Z","started_at":"2026-09-17T15:38:04Z","completed_at":"2026-09-17T15:42:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.31226407Z","timing_metadata_checked_at":"2026-09-18T15:41:15.31226407Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:15.31779519Z","region_metadata_checked_at":"2026-09-19T03:30:40.507019443Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"9c96aecc8c2d0af5c8dbe0e76590f76a0f2e598bfe4f6e2a7d28bf11316f371a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100610270391963648","occurred_at":"2026-09-17T15:38:04Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100609583708901376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:35:26Z","started_at":"2026-09-17T15:35:26Z","completed_at":"2026-09-17T15:35:47Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:28.177284501Z","timing_metadata_checked_at":"2026-09-18T15:41:28.177284501Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:28.180493812Z","region_metadata_checked_at":"2026-09-19T03:30:45.585103547Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ead3c720b3c31cde6ac4417441b706e0d5fdf78b7a431fcc750650359168691e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100609583708901376","occurred_at":"2026-09-17T15:35:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100609192539721728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6998,"pr_state":"closed","pr_sha":"c947fb99750b0318d83264c65a0eebdbade0c5e1","final_merged_sha":"c947fb99750b0318d83264c65a0eebdbade0c5e1","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:33:53Z","started_at":"2026-09-17T15:33:53Z","completed_at":"2026-09-17T15:46:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:17.812706016Z","timing_metadata_checked_at":"2026-09-18T15:41:17.812706016Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:17.81941574Z","region_metadata_checked_at":"2026-09-19T03:30:40.903036194Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8e144692ae2dd899adf01a79d25b740edc241ed88c20a3eced103f3e4e5fc5df","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100609192539721728","occurred_at":"2026-09-17T15:33:53Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100608785948086272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:32:16Z","started_at":"2026-09-17T15:32:16Z","completed_at":"2026-09-17T15:32:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:28.069310411Z","timing_metadata_checked_at":"2026-09-18T15:41:28.069310411Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:28.072488123Z","region_metadata_checked_at":"2026-09-19T03:30:45.397158159Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"df92afbc53405fb992a34c41ddef4102ec48ad4ac3e2229fabbc31254ba9a673","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100608785948086272","occurred_at":"2026-09-17T15:32:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100608561141780480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:31:19Z","started_at":"2026-09-17T15:31:19Z","completed_at":"2026-09-17T15:44:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:24.018781706Z","timing_metadata_checked_at":"2026-09-18T15:40:24.018781706Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:24.021845717Z","region_metadata_checked_at":"2026-09-19T03:30:34.72872057Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6deb2c7403a3f0e3ca79ec41b53203c154559a2370ca2f6b60213b76dafadf44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100608561141780480","occurred_at":"2026-09-17T15:31:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100607896134881280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:28:39Z","started_at":"2026-09-17T15:28:39Z","completed_at":"2026-09-17T15:29:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:27.945451865Z","timing_metadata_checked_at":"2026-09-18T15:41:27.945451865Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:27.948971277Z","region_metadata_checked_at":"2026-09-19T03:30:45.299685102Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ed83086085b8ed6cbd87a585e35d689fef40e7bdcffb497a1faa4d540269c53e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100607896134881280","occurred_at":"2026-09-17T15:28:39Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100607692493033472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"b6b00cdc8ea3ee249a9faa811955e4eb095652ef","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:27:55Z","started_at":"2026-09-17T15:27:55Z","completed_at":"2026-09-17T15:31:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:08.465963344Z","timing_metadata_checked_at":"2026-09-18T15:40:08.465963344Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:08.47045046Z","region_metadata_checked_at":"2026-09-19T03:30:32.83035967Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c8489d3791708c8509e3cab13c09ac40bb5c010f1e3989edcfd005e09d0cb1d4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100607692493033472","occurred_at":"2026-09-17T15:27:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100606978962231296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:25:05Z","started_at":"2026-09-17T15:25:05Z","completed_at":"2026-09-17T15:25:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:27.821637618Z","timing_metadata_checked_at":"2026-09-18T15:41:27.821637618Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:27.824778629Z","region_metadata_checked_at":"2026-09-19T03:30:45.20094824Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"83bb6e617e9c591c0d77f11fdec4045154581867fe448f5d9604d6a84cb3ba8e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100606978962231296","occurred_at":"2026-09-17T15:25:05Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100606027715055616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:21:19Z","started_at":"2026-09-17T15:21:19Z","completed_at":"2026-09-17T15:21:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:27.624812108Z","timing_metadata_checked_at":"2026-09-18T15:41:27.624812108Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:27.62794872Z","region_metadata_checked_at":"2026-09-19T03:30:45.107834199Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"d40d58d597a3ae0bc2db99d4f5756f541964079b0ff3470a02189118f8cae6e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100606027715055616","occurred_at":"2026-09-17T15:21:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100605726014574592","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6965,"pr_state":"closed","pr_sha":"88cc31ebdf51e981ac966aab5fe4333c0fee21a7","final_merged_sha":"df3789994ca1fe75767adc864f7cd6665c66bbef","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:20:07Z","started_at":"2026-09-17T15:20:07Z","completed_at":"2026-09-17T15:23:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:59.912684429Z","timing_metadata_checked_at":"2026-09-18T15:40:59.912684429Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:59.91579384Z","region_metadata_checked_at":"2026-09-19T03:30:38.572364444Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"c5e884f876c0d451644e2d9a430997a05f3c7a969af3ffbff5f249a6c625ca18","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100605726014574592","occurred_at":"2026-09-17T15:20:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100605242390351872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:18:11Z","started_at":"2026-09-17T15:18:11Z","completed_at":"2026-09-17T15:18:42Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:27.53376908Z","timing_metadata_checked_at":"2026-09-18T15:41:27.53376908Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:27.537391793Z","region_metadata_checked_at":"2026-09-19T03:30:45.00980564Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"17e20817842cb417dac0ff07718f1896dcd6eb1fe78a73fb4882d391290c6f88","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100605242390351872","occurred_at":"2026-09-17T15:18:11Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100605111339323392","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6856,"pr_state":"closed","pr_sha":"3488ce75b90ddf477eaba7e960e23f8d4d5145bc","final_merged_sha":"96f38fd5a277adea0aeec323ef5685124af19024","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:17:40Z","started_at":"2026-09-17T15:17:40Z","completed_at":"2026-09-17T15:21:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:28.973108531Z","timing_metadata_checked_at":"2026-09-18T15:40:28.973108531Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:28.976213142Z","region_metadata_checked_at":"2026-09-19T03:30:35.354915679Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b518a51cb9ce0ef3626a403f52efe92929da68e54739b02af3b8a92b705d8627","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100605111339323392","occurred_at":"2026-09-17T15:17:40Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100604282289000448","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:14:20Z","started_at":"2026-09-17T15:14:20Z","completed_at":"2026-09-17T15:14:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:27.373953403Z","timing_metadata_checked_at":"2026-09-18T15:41:27.373953403Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:27.377133715Z","region_metadata_checked_at":"2026-09-19T03:30:44.862082998Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ea20274f2431ac75772f4f7fead8b5fbc7e187f921453ee1fd951d8088e7538c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100604282289000448","occurred_at":"2026-09-17T15:14:20Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100603794415947776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6807,"pr_state":"open","pr_sha":"257db47c10ddb67d7539ad624828ade0c814c1ee","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:12:26Z","started_at":"2026-09-17T15:12:26Z","completed_at":"2026-09-17T15:18:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:24.729071362Z","timing_metadata_checked_at":"2026-09-18T15:40:24.729071362Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:24.732292573Z","region_metadata_checked_at":"2026-09-19T03:30:35.022081152Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ab482a83c4b5c44a2e718e6d2202a4e0f199d400f684f03261b639ecee379e92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6807/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100603794415947776","occurred_at":"2026-09-17T15:12:26Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100603648819073024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"bf99de6ce54fd35c709ee3aa6fb1c4efe2be125a","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:11:51Z","started_at":"2026-09-17T15:11:51Z","completed_at":"2026-09-17T15:18:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:23.895750863Z","timing_metadata_checked_at":"2026-09-18T15:40:23.895750863Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:23.899846578Z","region_metadata_checked_at":"2026-09-19T03:30:34.628767301Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b015fd809fe54eae46d41b8ab9afaa888b1e33e2ecbca0089b44adfaae8f584d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100603648819073024","occurred_at":"2026-09-17T15:11:51Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100601484545626112","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6999,"pr_state":"closed","pr_sha":"22907b2c6676e8ced797d116777be25a5231dfba","final_merged_sha":"91261d5b93a8d4039291003288ddae318a2a8506","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T15:03:04Z","started_at":"2026-09-17T15:03:04Z","completed_at":"2026-09-17T15:14:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:19.500660529Z","timing_metadata_checked_at":"2026-09-18T15:41:19.500660529Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:19.504585843Z","region_metadata_checked_at":"2026-09-19T03:30:41.005501469Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6504665fac70817d0cf3bec74d3d23a534c3ee981ae3c6f8190f6c2e801d68dd","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100601484545626112","occurred_at":"2026-09-17T15:03:04Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100601414920179712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T15:02:59Z","started_at":"2026-09-17T15:02:59Z","completed_at":"2026-09-17T15:12:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:27.258179986Z","timing_metadata_checked_at":"2026-09-18T15:41:27.258179986Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:27.261828899Z","region_metadata_checked_at":"2026-09-19T03:30:44.768927957Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"4b84de728df28c4d8068f34324a3f133e2183898c0f1ebcc3a43ed3db0505342","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100601414920179712","occurred_at":"2026-09-17T15:02:59Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100599858778869760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:56:48Z","started_at":"2026-09-17T14:56:48Z","completed_at":"2026-09-17T15:05:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:02.617203365Z","timing_metadata_checked_at":"2026-09-18T15:41:02.617203365Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:02.622050682Z","region_metadata_checked_at":"2026-09-19T03:30:39.145049853Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"de751fa7b794f86dcb9e17d79250216f727d6196762107c53a1ac15a4498080c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100599858778869760","occurred_at":"2026-09-17T14:56:48Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100599384889626624","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:54:46Z","started_at":"2026-09-17T14:54:46Z","completed_at":"2026-09-17T15:00:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:43.912388779Z","timing_metadata_checked_at":"2026-09-18T15:40:43.912388779Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:43.91558459Z","region_metadata_checked_at":"2026-09-19T03:30:37.043986308Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"96299980ae88b9729c73fec9a8ee46a36d0532a938f47a25b75150c9bd32f8b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100599384889626624","occurred_at":"2026-09-17T14:54:46Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100598388113281024","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:50:57Z","started_at":"2026-09-17T14:50:57Z","completed_at":"2026-09-17T14:59:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:27.138241853Z","timing_metadata_checked_at":"2026-09-18T15:41:27.138241853Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:27.141279564Z","region_metadata_checked_at":"2026-09-19T03:30:44.664793076Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"90ce7c0991912d03bc4345a013ca8ff4f9cd4b327a159a7a6e77eb12c5fd11f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100598388113281024","occurred_at":"2026-09-17T14:50:57Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100592680860913664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:28:16Z","started_at":"2026-09-17T14:28:16Z","completed_at":"2026-09-17T14:28:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:27.011470896Z","timing_metadata_checked_at":"2026-09-18T15:41:27.011470896Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:27.014699008Z","region_metadata_checked_at":"2026-09-19T03:30:44.571028832Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e5d9edf7a505894fab9d4e0d37c0ba19a3aef73210c2fb36c43ea5b2f89c7172","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100592680860913664","occurred_at":"2026-09-17T14:28:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6997/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100591856864727040","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6997,"pr_state":"open","pr_sha":"8f7f929be6e33a90811214fb7e9a90b0edc1abcc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T14:24:37Z","started_at":"2026-09-17T14:24:37Z","completed_at":"2026-09-17T14:32:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.946793063Z","timing_metadata_checked_at":"2026-09-18T15:41:15.946793063Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:15.9568964Z","region_metadata_checked_at":"2026-09-19T03:30:40.805406136Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"68fb2ace4ed62535925b88b8f5ba2cc631a5091b1630fa154f7198776b7f49e4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6997/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100591856864727040","occurred_at":"2026-09-17T14:24:37Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100591817819951104","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:24:30Z","started_at":"2026-09-17T14:24:30Z","completed_at":"2026-09-17T14:25:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:26.868844982Z","timing_metadata_checked_at":"2026-09-18T15:41:26.868844982Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:26.872504995Z","region_metadata_checked_at":"2026-09-19T03:30:44.463676439Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5484d1d57df2464c8a2eb45828cce64aef43e76b2bcf9e39279e5b6923ae6df6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100591817819951104","occurred_at":"2026-09-17T14:24:30Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100590729666498560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6580,"pr_state":"open","pr_sha":"ca657d4d97703c8bad13bcf1f6f446ac313cf1bc","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T14:20:31Z","started_at":"2026-09-17T14:20:31Z","completed_at":"2026-09-17T14:26:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:09.018281233Z","timing_metadata_checked_at":"2026-09-18T15:40:09.018281233Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:09.021873346Z","region_metadata_checked_at":"2026-09-19T03:30:33.131245079Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"68072b3b962a8d1042c06c459ab34cdd1bbc1de431032ccd6554dc2ce285dd87","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6580/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100590729666498560","occurred_at":"2026-09-17T14:20:31Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100589824825102336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:16:55Z","started_at":"2026-09-17T14:16:55Z","completed_at":"2026-09-17T14:17:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:26.697051462Z","timing_metadata_checked_at":"2026-09-18T15:41:26.697051462Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:26.700083273Z","region_metadata_checked_at":"2026-09-19T03:30:44.351073026Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b970e4a04557aa07cddb1f5750ba242dc1bad67fd105e44aef3a86ace5bba73c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100589824825102336","occurred_at":"2026-09-17T14:16:55Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100589446649876480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:15:22Z","started_at":"2026-09-17T14:15:22Z","completed_at":"2026-09-17T14:26:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.931297145Z","timing_metadata_checked_at":"2026-09-18T15:40:44.931297145Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:44.934578457Z","region_metadata_checked_at":"2026-09-19T03:30:37.519898763Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"b39298a56c1c3b2b896611cd734d477ce2943e1e4453c98ef3bfa43478610767","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100589446649876480","occurred_at":"2026-09-17T14:15:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100589221554163712","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6985,"pr_state":"closed","pr_sha":"d7b0b57bd15ef0bbc4edf6ad998bd65072d074ae","final_merged_sha":"d7b0b57bd15ef0bbc4edf6ad998bd65072d074ae","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:14:28Z","started_at":"2026-09-17T14:14:28Z","completed_at":"2026-09-17T14:20:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:09.762879911Z","timing_metadata_checked_at":"2026-09-18T15:41:09.762879911Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:09.766589224Z","region_metadata_checked_at":"2026-09-19T03:30:40.087973707Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"44ae41bdfdf053aaf55bcdd5d742f83393616048ceaf69747fe18284256a77da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100589221554163712","occurred_at":"2026-09-17T14:14:28Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100589048190996480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6880,"pr_state":"closed","pr_sha":"2e36726e4a97495428698b756402558a81b5586a","final_merged_sha":"a20dd280dd6b7499cf853051219487314ef1e460","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T14:13:49Z","started_at":"2026-09-17T14:13:49Z","completed_at":"2026-09-17T14:18:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:31.087040433Z","timing_metadata_checked_at":"2026-09-18T15:40:31.087040433Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:31.090513546Z","region_metadata_checked_at":"2026-09-19T03:30:35.692764625Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"2fcb879017da5528c8035da40308ac8e5f1300b820ba234e44d2c74ae6244946","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100589048190996480","occurred_at":"2026-09-17T14:13:49Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100588858616844288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"3f0521ff633af0368f049bfead3093e31565b50f","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:13:02Z","started_at":"2026-09-17T14:13:02Z","completed_at":"2026-09-17T14:18:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:35.567125445Z","timing_metadata_checked_at":"2026-09-18T15:40:35.567125445Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:35.572497265Z","region_metadata_checked_at":"2026-09-19T03:30:36.174560302Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"5b21dd7962c5479f9797ed16c1c81017c049c6691af87c932602323313dfd532","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100588858616844288","occurred_at":"2026-09-17T14:13:02Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6997/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100588476914208768","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6997,"pr_state":"open","pr_sha":"11ad8964a023494cb41456e17b7f927b86c681bd","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T14:09:02Z","started_at":"2026-09-17T14:09:02Z","completed_at":"2026-09-17T14:24:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.825756626Z","timing_metadata_checked_at":"2026-09-18T15:41:15.825756626Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:15.829191038Z","region_metadata_checked_at":"2026-09-19T03:30:40.709642385Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ced06636c854f325c30a5b89c44752ebca42caa643ea169d7ea65d08476cc035","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6997/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100588476914208768","occurred_at":"2026-09-17T14:09:02Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100588090610421761","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T14:08:57Z","started_at":"2026-09-17T14:08:57Z","completed_at":"2026-09-17T14:13:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:26.598706307Z","timing_metadata_checked_at":"2026-09-18T15:41:26.598706307Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:26.602548121Z","region_metadata_checked_at":"2026-09-19T03:30:44.249543854Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8bb05785e8dae2c3b025c74d0219ee1f7cb641cbe91441d787787f089e3c0b94","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100588090610421761","occurred_at":"2026-09-17T14:08:57Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100588377932828672","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6157,"pr_state":"open","pr_sha":"d4af5249d6e0478ea45dc795343aaa530c0f147b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T13:59:10Z","started_at":"2026-09-17T13:59:10Z","completed_at":"2026-09-17T14:31:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:04.748385858Z","timing_metadata_checked_at":"2026-09-18T15:40:04.748385858Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:04.751450069Z","region_metadata_checked_at":"2026-09-19T03:30:32.45621369Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"18a2f2a6d1916ec1e402adec3fdf277c7993aff34ad44604f55a7bb8485f061f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100588377932828672","occurred_at":"2026-09-17T13:59:10Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100576592332853248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6988,"pr_state":"closed","pr_sha":"53b8fd9bee7c04069b9537ac28ccedffde355125","final_merged_sha":"53b8fd9bee7c04069b9537ac28ccedffde355125","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T13:24:04Z","started_at":"2026-09-17T13:24:04Z","completed_at":"2026-09-17T13:27:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.022557379Z","timing_metadata_checked_at":"2026-09-18T15:41:12.022557379Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:41:12.025858191Z","region_metadata_checked_at":"2026-09-19T03:30:40.201755124Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"da0c779dde3198bb1c9c9594af8309f754dd9eb3964a761af904574324004baf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100576592332853248","occurred_at":"2026-09-17T13:24:04Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100573010787307520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6985,"pr_state":"closed","pr_sha":"ff968d9cec79747e162d6e00c389602dc3e02ef4","final_merged_sha":"d7b0b57bd15ef0bbc4edf6ad998bd65072d074ae","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T13:10:07Z","started_at":"2026-09-17T13:10:07Z","completed_at":"2026-09-17T14:05:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:09.655542123Z","timing_metadata_checked_at":"2026-09-18T15:41:09.655542123Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:09.658808535Z","region_metadata_checked_at":"2026-09-18T15:41:09.658808535Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"590f203a99587233cbe134b17d9102aa8abafa8e4fecb28edf0e0296717c2459","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100573010787307520","occurred_at":"2026-09-17T13:10:07Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100570778985566208","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T13:01:15Z","started_at":"2026-09-17T13:01:15Z","completed_at":"2026-09-17T14:02:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:34.678131448Z","timing_metadata_checked_at":"2026-09-18T15:40:34.678131448Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:34.68131966Z","region_metadata_checked_at":"2026-09-18T15:40:34.68131966Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"8a06d472deb8d0614d8b03a4957e08b040f1151323c720fafcca15e00d5e4fed","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100570778985566208","occurred_at":"2026-09-17T13:01:15Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"764ee60fa32d166f92c6c286f96007787ee189de","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T12:57:19Z","started_at":"2026-09-17T12:57:19Z","completed_at":"2026-09-17T14:03:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:23.799695418Z","timing_metadata_checked_at":"2026-09-18T15:40:23.799695418Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:23.802944529Z","region_metadata_checked_at":"2026-09-18T15:40:23.802944529Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"1eeb58e696fe64e69535316b33b516a1cb269b068a957584ef908c48ec5f8d74","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","failed_at":"e2e","signature_id":"f24c215311a297177eaa94e5511fcc51ba57d864ab61313b4419552b1cf74930","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb8ee0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cjljbxb6clv6\" to finish creating: GET https://rp.j6143232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/631b27eb-736d-4b48-8f88-11668c9ed067\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/631b27eb-736d-4b48-8f88-11668c9ed067\",\n \"name\": \"631b27eb-736d-4b48-8f88-11668c9ed067\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T14:02:04.639808723Z\",\n \"endTime\": \"2026-09-17T14:31:12.3306894Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 24m33s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000eb8ee0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-cjljbxb6clv6\" to finish creating: GET https://rp.j6143232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/631b27eb-736d-4b48-8f88-11668c9ed067\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/631b27eb-736d-4b48-8f88-11668c9ed067\",\n \"name\": \"631b27eb-736d-4b48-8f88-11668c9ed067\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T14:02:04.639808723Z\",\n \"endTime\": \"2026-09-17T14:31:12.3306894Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 24m33s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a2a8c1a1821a143ad9d17f7f03a34cbdd719fd3ffa8dcd67175fce3ae9856aa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","failed_at":"e2e","signature_id":"824964c0c575d45ef044ea907156d5bdd2f98d69978b43f2f1fb3b622414875f","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001092960\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-n6bjtxs9xn9r\" to finish creating: GET https://rp.j6143232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eda415e4-abae-48ea-8897-a619811ef2d8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eda415e4-abae-48ea-8897-a619811ef2d8\",\n \"name\": \"eda415e4-abae-48ea-8897-a619811ef2d8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T14:01:55.971807004Z\",\n \"endTime\": \"2026-09-17T14:31:02.419340074Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m49s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001092960\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-n6bjtxs9xn9r\" to finish creating: GET https://rp.j6143232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eda415e4-abae-48ea-8897-a619811ef2d8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eda415e4-abae-48ea-8897-a619811ef2d8\",\n \"name\": \"eda415e4-abae-48ea-8897-a619811ef2d8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T14:01:55.971807004Z\",\n \"endTime\": \"2026-09-17T14:31:02.419340074Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m49s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d133d8538f99a4fb115db9fc61bb35828e9465b92b19f83ebde4a41de93fd225","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","failed_at":"e2e","signature_id":"62a250c534c20b5ad145d249c89f89739cbae0c9b5704a8277cba737ff1f0047","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000889420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-nwn99xtp6np4\" to finish creating: GET https://rp.j6143232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d8268f9-6e03-421e-aa07-d0427e5d6f85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d8268f9-6e03-421e-aa07-d0427e5d6f85\",\n \"name\": \"7d8268f9-6e03-421e-aa07-d0427e5d6f85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T14:00:30.716316995Z\",\n \"endTime\": \"2026-09-17T14:29:32.422264706Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 24m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000889420\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-nwn99xtp6np4\" to finish creating: GET https://rp.j6143232.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d8268f9-6e03-421e-aa07-d0427e5d6f85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d8268f9-6e03-421e-aa07-d0427e5d6f85\",\n \"name\": \"7d8268f9-6e03-421e-aa07-d0427e5d6f85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T14:00:30.716316995Z\",\n \"endTime\": \"2026-09-17T14:29:32.422264706Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 24m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569777780035584","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5724,"pr_state":"closed","pr_sha":"5e814b754c6ac49bdda39af9047f424419d1397c","final_merged_sha":"02f55855ad2e600a742ce1b551002c7fbd872350","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T12:57:16Z","started_at":"2026-09-17T12:57:16Z","completed_at":"2026-09-17T14:01:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:03.681131719Z","timing_metadata_checked_at":"2026-09-18T15:40:03.681131719Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:03.684626731Z","region_metadata_checked_at":"2026-09-18T15:40:03.684626731Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"75412a84c46810af112cc37903b607cf60ff957c29f1aef07337109c3f21d65e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569777780035584","occurred_at":"2026-09-17T12:57:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6989,"pr_state":"closed","pr_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","final_merged_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T12:57:11Z","started_at":"2026-09-17T12:57:11Z","completed_at":"2026-09-17T14:00:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.688037384Z","timing_metadata_checked_at":"2026-09-18T15:41:12.688037384Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:12.691451596Z","region_metadata_checked_at":"2026-09-18T15:41:12.691451596Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"2848217c0d6a46356c7afa56257cfd6bc5ce40c4d6ad4744c025ac1544412bd1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","failed_at":"e2e","signature_id":"b4cc553a2220034e450c979876c39e29fdc1278faa5ca56fae473d7a07ad2b7c","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc0014ee660\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n \u003c*errors.joinError | 0xc0014ee660\u003e: \n VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7855bddb074e5ee4da59762c697f802a687000bfd9477e9f92fc724e190b9330","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-\u003cid\u003e in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)"}},{"row_id":"6bce776715412c0c0905cc54e62b99c6db480683e360eccb91d5340b1f272a14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","failed_at":"e2e","signature_id":"903f3f622deb52a1f1d8dea7e418d3ea7d8db5819b70eccb79ae8ea0a5299dd4","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001102400\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-fq99lts22lbf\" to finish creating: GET https://rp.j5074560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/efdd5ddb-35ed-4306-8558-b11e4c1ba83c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/efdd5ddb-35ed-4306-8558-b11e4c1ba83c\",\n \"name\": \"efdd5ddb-35ed-4306-8558-b11e4c1ba83c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T14:03:49.544259109Z\",\n \"endTime\": \"2026-09-17T14:32:53.332127754Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001102400\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-fq99lts22lbf\" to finish creating: GET https://rp.j5074560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/efdd5ddb-35ed-4306-8558-b11e4c1ba83c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/efdd5ddb-35ed-4306-8558-b11e4c1ba83c\",\n \"name\": \"efdd5ddb-35ed-4306-8558-b11e4c1ba83c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T14:03:49.544259109Z\",\n \"endTime\": \"2026-09-17T14:32:53.332127754Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a06fc9f7cc659f298e308135c3269b1f0be585513e5716b3f8a1f0d26a60a20","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","failed_at":"alert","signature_id":"2b92ea2834b8405b970ceee37d04d32b047f88649536fb5e87c255a7cf6b9715","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T14:14:45Z\n Ended: 2026-09-17T14:44:45Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5074560-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssqahfphn079fl5988maka6l79507m4-private-ingress\", pod=\"router-6d6447d77d-xlpbk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssqahfphn079fl5988maka6l79507m4-private-ingress/router-6d6447d77d-xlpbk has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["7855bddb074e5ee4da59762c697f802a687000bfd9477e9f92fc724e190b9330","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569750479310848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6879,"pr_state":"closed","pr_sha":"865e554719be3a2a33ad3c12d3fddff783af5356","final_merged_sha":"865e554719be3a2a33ad3c12d3fddff783af5356","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T12:57:09Z","started_at":"2026-09-17T12:57:09Z","completed_at":"2026-09-17T14:04:04Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:29.613357333Z","timing_metadata_checked_at":"2026-09-18T15:40:29.613357333Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:29.616840446Z","region_metadata_checked_at":"2026-09-18T15:40:29.616840446Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"6adea4b632ce33906dab1839f62fb2205823ccb5fa2ab077851afa39db12a58a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569750479310848","occurred_at":"2026-09-17T12:57:09Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569730229211136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6986,"pr_state":"closed","pr_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","final_merged_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T12:57:04Z","started_at":"2026-09-17T12:57:04Z","completed_at":"2026-09-17T14:01:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:11.61075909Z","timing_metadata_checked_at":"2026-09-18T15:41:11.61075909Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:11.614011202Z","region_metadata_checked_at":"2026-09-18T15:41:11.614011202Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"76be1a199471c8cac7875670d361eef32f369fb7db5c0abc99882bd44f7d2136","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569730229211136","occurred_at":"2026-09-17T12:57:04Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569681449455616","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6157,"pr_state":"open","pr_sha":"cb3d4e25f9f223901db08108e0232997a8c1c6b8","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T12:56:50Z","started_at":"2026-09-17T12:56:50Z","completed_at":"2026-09-17T14:01:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:04.660784543Z","timing_metadata_checked_at":"2026-09-18T15:40:04.660784543Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:04.663950754Z","region_metadata_checked_at":"2026-09-18T15:40:04.663950754Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"0d995bf347ebcdd84bdfb37d1f774f6a8b3300a58dedd60896f17990ea775576","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569681449455616","occurred_at":"2026-09-17T12:56:50Z","failed_at":"e2e","signature_id":"2126196a18f3d500d8a52360e541d2fceb67e8a67c46b2636cf54189a72bfe35","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000d270f8\u003e: \n failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000d270f8\u003e: \n failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d33373d20e5996675bec12509adcc946f8d57b13e0f18cc57679d39e08115eb4","failure_pattern":"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"\u003cresource-group\u003e\": Get \"\u003curl\u003e\": stream error: stream ID 13; CANCEL; received from peer","search_query":"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d33373d20e5996675bec12509adcc946f8d57b13e0f18cc57679d39e08115eb4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6886,"pr_state":"open","pr_sha":"a6aa6a03a137ec55e3f4f5d06cc620e6bf6dd7f3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T12:45:27Z","started_at":"2026-09-17T12:45:27Z","completed_at":"2026-09-17T14:05:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:32.497941307Z","timing_metadata_checked_at":"2026-09-18T15:40:32.497941307Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:32.501059619Z","region_metadata_checked_at":"2026-09-18T15:40:32.501059619Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"8613dc5fa19009022090dc8bd74021177c701b5d2c765a734f5d0ccc4f376110","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","failed_at":"e2e","signature_id":"e828cec50a4c240181974579c6679994f89f3d798f5316edd2d635df29854690","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1d120\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qmzbhsdszqnv\" to finish creating: GET https://rp.j8995200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dbbd6da6-c17e-49e2-af9e-2a3933f43a85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dbbd6da6-c17e-49e2-af9e-2a3933f43a85\",\n \"name\": \"dbbd6da6-c17e-49e2-af9e-2a3933f43a85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:28:36.437510186Z\",\n \"endTime\": \"2026-09-17T13:57:39.205388869Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c1d120\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-qmzbhsdszqnv\" to finish creating: GET https://rp.j8995200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dbbd6da6-c17e-49e2-af9e-2a3933f43a85\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dbbd6da6-c17e-49e2-af9e-2a3933f43a85\",\n \"name\": \"dbbd6da6-c17e-49e2-af9e-2a3933f43a85\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:28:36.437510186Z\",\n \"endTime\": \"2026-09-17T13:57:39.205388869Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ac3bb396e6a261bfc8831ab734607c181aa92eb61398b2d02f1c26187a6138e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","failed_at":"e2e","signature_id":"1fb4041dc59d81c7e183948830ae7ac1b2783b718bf3d7631b04090e55e3f847","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000846b80\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-l4rcq7cd4xkr\" to finish creating: GET https://rp.j8995200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fb12334b-a512-43ea-824e-be2e5c1b7445\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fb12334b-a512-43ea-824e-be2e5c1b7445\",\n \"name\": \"fb12334b-a512-43ea-824e-be2e5c1b7445\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:27:46.68038337Z\",\n \"endTime\": \"2026-09-17T13:56:49.088605231Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000846b80\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-l4rcq7cd4xkr\" to finish creating: GET https://rp.j8995200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fb12334b-a512-43ea-824e-be2e5c1b7445\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fb12334b-a512-43ea-824e-be2e5c1b7445\",\n \"name\": \"fb12334b-a512-43ea-824e-be2e5c1b7445\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:27:46.68038337Z\",\n \"endTime\": \"2026-09-17T13:56:49.088605231Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cad4cb16f86fbd751e0a29802ab5ce674626062c2cc0fff01cc484c78910da40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","failed_at":"alert","signature_id":"4815c5ee092aaeb9f17a4f179de41c68c528994add0634df56951333903777fc","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:43:06Z\n Ended: 2026-09-17T14:08:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8995200-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspq1c5fekjjfpb2eogshu5ujqg9qiq-z1f6t4a0j5i6b3o\", pod=\"router-58bb4b4c4b-qx4x4\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspq1c5fekjjfpb2eogshu5ujqg9qiq-z1f6t4a0j5i6b3o/router-58bb4b4c4b-qx4x4 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"f865c0dfffe96ecf0c3a5121741da346f6e70fadba7811ef08dd7b1ad8eefaba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","failed_at":"e2e","signature_id":"3a922f4d19da47f001e6311e71c0f1a6d255911648780e0fdc7768301e19e9b8","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010622c0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-2k9hnrc7s8nw\" to finish creating: GET https://rp.j8995200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13dbceb4-6c92-4d49-97f3-f75c8277895b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13dbceb4-6c92-4d49-97f3-f75c8277895b\",\n \"name\": \"13dbceb4-6c92-4d49-97f3-f75c8277895b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:28:50.480676293Z\",\n \"endTime\": \"2026-09-17T13:57:59.187242674Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010622c0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-2k9hnrc7s8nw\" to finish creating: GET https://rp.j8995200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13dbceb4-6c92-4d49-97f3-f75c8277895b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13dbceb4-6c92-4d49-97f3-f75c8277895b\",\n \"name\": \"13dbceb4-6c92-4d49-97f3-f75c8277895b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:28:50.480676293Z\",\n \"endTime\": \"2026-09-17T13:57:59.187242674Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6979,"pr_state":"open","pr_sha":"51d3c396ae56e46e8d0d671b4b44e52eb7b16d83","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T12:31:48Z","started_at":"2026-09-17T12:31:48Z","completed_at":"2026-09-17T14:04:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:06.912737533Z","timing_metadata_checked_at":"2026-09-18T15:41:06.912737533Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:06.916449246Z","region_metadata_checked_at":"2026-09-18T15:41:06.916449246Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0dadb8f439e091c0cb70b6ea270bfe506de672882ec72474d0cd1e87f4d17a73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","failed_at":"alert","signature_id":"ece77cd8e3e47a8b500cc23dba72bce96033933da8cfc561a7bfe5d93a950f77","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:22:10Z\n Ended: 2026-09-17T13:38:09Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2532096-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssphmmqqlnjk70h22r3i4bauan32tpt-e5j6t5h1p6n1t9u\", pod=\"router-669df879db-lfkh8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssphmmqqlnjk70h22r3i4bauan32tpt-e5j6t5h1p6n1t9u/router-669df879db-lfkh8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"5db1358192196a2213a437685434bdda7a14b4b7ccdaa1e6e0a85f1fa3742140","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","failed_at":"e2e","signature_id":"9771d1687cb82ab4b14c0f80173555cf109826201233928e90423e0171bc2724","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-pxkzk5k6tj2b/cluster-candidate-4-20-rdptdx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001252760\u003e: \n failed to create HCP cluster cluster-candidate-4-20-rdptdx: failed waiting for cluster=\"cluster-candidate-4-20-rdptdx\" in resourcegroup=\"rg-candidate-4-20-pxkzk5k6tj2b\" to finish creating: GET https://rp.j2532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ce05291f-4663-43c2-95b1-34c4d5d56d3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ce05291f-4663-43c2-95b1-34c4d5d56d3c\",\n \"name\": \"ce05291f-4663-43c2-95b1-34c4d5d56d3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:10:50.219697768Z\",\n \"endTime\": \"2026-09-17T13:39:57.0065588Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 23m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-pxkzk5k6tj2b/cluster-candidate-4-20-rdptdx should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001252760\u003e: \n failed to create HCP cluster cluster-candidate-4-20-rdptdx: failed waiting for cluster=\"cluster-candidate-4-20-rdptdx\" in resourcegroup=\"rg-candidate-4-20-pxkzk5k6tj2b\" to finish creating: GET https://rp.j2532096.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ce05291f-4663-43c2-95b1-34c4d5d56d3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ce05291f-4663-43c2-95b1-34c4d5d56d3c\",\n \"name\": \"ce05291f-4663-43c2-95b1-34c4d5d56d3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:10:50.219697768Z\",\n \"endTime\": \"2026-09-17T13:39:57.0065588Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 23m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"af0f20eaac617e955d49a39790583a0efa40d8220cbdb8b3487468e92c10c1d1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","failed_at":"alert","signature_id":"e9c0e585944f16bc753c7dace1676a24a15848f5a821759ef30ea4b61ef466bc","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:42:29Z\n Ended: 2026-09-17T13:48:30Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2532096-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2532096-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T12:23:47Z","started_at":"2026-09-17T12:23:47Z","completed_at":"2026-09-17T14:05:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:26.485070098Z","timing_metadata_checked_at":"2026-09-18T15:41:26.485070098Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:26.48860881Z","region_metadata_checked_at":"2026-09-18T15:41:26.48860881Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"4a61e49f418e7d4cc0be64ad82d1306e8e2e42a491aa8c5177f241f5252a8a36","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","failed_at":"e2e","signature_id":"cb6c78925476f970ec41ca2cc86883f3fe25eedebef129a57d35868eedaf968c","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xw728ddtqcc2/cluster-hs-pre-bht88r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe0440\u003e: \n failed to create HCP cluster cluster-hs-pre-bht88r: failed waiting for cluster=\"cluster-hs-pre-bht88r\" in resourcegroup=\"rg-hs-pre--xw728ddtqcc2\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99f56462-8a68-42f5-b39b-3881c0af36e7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99f56462-8a68-42f5-b39b-3881c0af36e7\",\n \"name\": \"99f56462-8a68-42f5-b39b-3881c0af36e7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:14:13.9724219Z\",\n \"endTime\": \"2026-09-17T13:43:14.263460227Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m36s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--xw728ddtqcc2/cluster-hs-pre-bht88r should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe0440\u003e: \n failed to create HCP cluster cluster-hs-pre-bht88r: failed waiting for cluster=\"cluster-hs-pre-bht88r\" in resourcegroup=\"rg-hs-pre--xw728ddtqcc2\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99f56462-8a68-42f5-b39b-3881c0af36e7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/99f56462-8a68-42f5-b39b-3881c0af36e7\",\n \"name\": \"99f56462-8a68-42f5-b39b-3881c0af36e7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:14:13.9724219Z\",\n \"endTime\": \"2026-09-17T13:43:14.263460227Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m36s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"625f070a9f5023c69d0763d6b35dc9e9a256c411a657c57c5a0d271ecd6b6e19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","failed_at":"e2e","signature_id":"b8f768f8fc135387ca0e0f20dc96f1cad4999bade1e76fff9ca2066634cf47d3","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006f47e0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rqblsgkhsl67\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffb5ae05-311d-47a9-81e8-a78c76770695\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffb5ae05-311d-47a9-81e8-a78c76770695\",\n \"name\": \"ffb5ae05-311d-47a9-81e8-a78c76770695\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:14:59.825714146Z\",\n \"endTime\": \"2026-09-17T13:44:04.017647611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006f47e0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-rqblsgkhsl67\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffb5ae05-311d-47a9-81e8-a78c76770695\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffb5ae05-311d-47a9-81e8-a78c76770695\",\n \"name\": \"ffb5ae05-311d-47a9-81e8-a78c76770695\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:14:59.825714146Z\",\n \"endTime\": \"2026-09-17T13:44:04.017647611Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m0s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"85824ff0c80a70904d6b5a59f48c0335ed15129ca6e39cc9d76357a6f57aa18f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","failed_at":"e2e","signature_id":"ab6e130ec463d100706975248c724bd66e5d7d3642c918a3434eeabaf4b5aa60","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0a6a0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-n5scb8djfw2f\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a13212a1-a469-4833-ad96-5cf68dc1002e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a13212a1-a469-4833-ad96-5cf68dc1002e\",\n \"name\": \"a13212a1-a469-4833-ad96-5cf68dc1002e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:14:27.329387989Z\",\n \"endTime\": \"2026-09-17T13:43:34.147081491Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0a6a0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-n5scb8djfw2f\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a13212a1-a469-4833-ad96-5cf68dc1002e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a13212a1-a469-4833-ad96-5cf68dc1002e\",\n \"name\": \"a13212a1-a469-4833-ad96-5cf68dc1002e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:14:27.329387989Z\",\n \"endTime\": \"2026-09-17T13:43:34.147081491Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ac23b6b05cfdea944405ff9540448ed8052e718b2d28395b4a9ceeb4a83398ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","failed_at":"alert","signature_id":"e8c855a1e7025053c74a73b5e48c4e6e96debc9ba1c7d6b424796483466cdf6f","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:26:50Z\n Ended: 2026-09-17T13:43:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6691200-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspiucgjqnoq5us70um13m98tr1nrls-g3c5i5x9u6t6v1r\", pod=\"router-69779865cd-8hd8n\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspiucgjqnoq5us70um13m98tr1nrls-g3c5i5x9u6t6v1r/router-69779865cd-8hd8n has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"d94b5b1eab8b5fad62a04fd52af4c09627cf18120b7f70963a1a91b1a4acbc7f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","failed_at":"e2e","signature_id":"41e17c7a4eecb6c1ed4ee23560b530d6b6a42361cc6e3ef0250dbdef28f6d545","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c05990\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53121-\u003e172.30.0.10:53: read: connection refused\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc000c05990\u003e: \n clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53121-\u003e172.30.0.10:53: read: connection refused\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02"}},{"row_id":"dc20d5f59fd36e20d02c7f4915d643381bbe7cf13d32629ea615a79700a3f560","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","failed_at":"e2e","signature_id":"ff4a343919b36fa3670725111608ea3b35a12089dab63a806e3c5e121a1fbb9a","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-tsxbd6f5jxgs/cluster-candidate-4-21-6rv67k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017be420\u003e: \n failed to create HCP cluster cluster-candidate-4-21-6rv67k: failed waiting for cluster=\"cluster-candidate-4-21-6rv67k\" in resourcegroup=\"rg-candidate-4-21-tsxbd6f5jxgs\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc291a9b-26c3-4ca5-924b-ef4256d35e27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc291a9b-26c3-4ca5-924b-ef4256d35e27\",\n \"name\": \"bc291a9b-26c3-4ca5-924b-ef4256d35e27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:14:50.324424123Z\",\n \"endTime\": \"2026-09-17T13:43:54.118510486Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-tsxbd6f5jxgs/cluster-candidate-4-21-6rv67k should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0017be420\u003e: \n failed to create HCP cluster cluster-candidate-4-21-6rv67k: failed waiting for cluster=\"cluster-candidate-4-21-6rv67k\" in resourcegroup=\"rg-candidate-4-21-tsxbd6f5jxgs\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc291a9b-26c3-4ca5-924b-ef4256d35e27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc291a9b-26c3-4ca5-924b-ef4256d35e27\",\n \"name\": \"bc291a9b-26c3-4ca5-924b-ef4256d35e27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:14:50.324424123Z\",\n \"endTime\": \"2026-09-17T13:43:54.118510486Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e2c603d936dc4d331ec975f731a6068ade5b44ddcd9958e3899543b58349f75b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","failed_at":"e2e","signature_id":"85532b20bf57446a7d41586b4163ec6f19b5563d86b6207d2b7d5ab6951e2651","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000884200\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-pqfbdsfxgpwf\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b7c6859-32e0-4c13-b03c-200c651d5ba9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b7c6859-32e0-4c13-b03c-200c651d5ba9\",\n \"name\": \"5b7c6859-32e0-4c13-b03c-200c651d5ba9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:15:12.801180509Z\",\n \"endTime\": \"2026-09-17T13:44:14.063627342Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m30s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000884200\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-pqfbdsfxgpwf\" to finish creating: GET https://rp.j6691200.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b7c6859-32e0-4c13-b03c-200c651d5ba9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5b7c6859-32e0-4c13-b03c-200c651d5ba9\",\n \"name\": \"5b7c6859-32e0-4c13-b03c-200c651d5ba9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:15:12.801180509Z\",\n \"endTime\": \"2026-09-17T13:44:14.063627342Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 21m30s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6995,"pr_state":"open","pr_sha":"f5f671665f064801eaa96e97102c3ac8357379c6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T12:18:54Z","started_at":"2026-09-17T12:18:54Z","completed_at":"2026-09-17T14:01:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.199085561Z","timing_metadata_checked_at":"2026-09-18T15:41:15.199085561Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:15.205173883Z","region_metadata_checked_at":"2026-09-18T15:41:15.205173883Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"56e6733480fd01800c50d91bad1f08b698a5a14c08a7699f836e2e86357f4159","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","failed_at":"e2e","signature_id":"4dab39d1b1d724fed25dc69d84d189cab2ac6a8152b77e555635e3621d133e55","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016a45a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jvkrg2m7vz86\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf1f66-7aa9-4611-bffb-22454b3ff87d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf1f66-7aa9-4611-bffb-22454b3ff87d\",\n \"name\": \"1daf1f66-7aa9-4611-bffb-22454b3ff87d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:09:14.976943123Z\",\n \"endTime\": \"2026-09-17T13:38:20.215505179Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016a45a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-jvkrg2m7vz86\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf1f66-7aa9-4611-bffb-22454b3ff87d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1daf1f66-7aa9-4611-bffb-22454b3ff87d\",\n \"name\": \"1daf1f66-7aa9-4611-bffb-22454b3ff87d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:09:14.976943123Z\",\n \"endTime\": \"2026-09-17T13:38:20.215505179Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5c97b85db2998feb2f62cf543aac8f297a98ef05f1914715fc101d0d1fb18aff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","failed_at":"e2e","signature_id":"ebcbd7286e42fab84ff7ea3f9b915f4d4683f1309800106ed6220b22cd9d54e2","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00061ecc0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9z5zhn4wcxq7\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/864bbcb9-28d5-49bd-9ab9-4a5ea088a8f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/864bbcb9-28d5-49bd-9ab9-4a5ea088a8f3\",\n \"name\": \"864bbcb9-28d5-49bd-9ab9-4a5ea088a8f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:10:00.288784653Z\",\n \"endTime\": \"2026-09-17T13:39:01.074596069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m44s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00061ecc0\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-9z5zhn4wcxq7\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/864bbcb9-28d5-49bd-9ab9-4a5ea088a8f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/864bbcb9-28d5-49bd-9ab9-4a5ea088a8f3\",\n \"name\": \"864bbcb9-28d5-49bd-9ab9-4a5ea088a8f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:10:00.288784653Z\",\n \"endTime\": \"2026-09-17T13:39:01.074596069Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m44s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"71d9420e8b23e40e2d70b90b931f88980b37ea7e00726d218ea4f5f87049932c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","failed_at":"e2e","signature_id":"39a1e24d4800633f0d0b42d4f18189154efcf362e824edcdb07cb8b8b0dd68d9","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df7160\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2mtbkjvnnzmc\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0607f3e-d3c6-4002-afc3-b216aabd4af7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0607f3e-d3c6-4002-afc3-b216aabd4af7\",\n \"name\": \"a0607f3e-d3c6-4002-afc3-b216aabd4af7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:09:52.721401557Z\",\n \"endTime\": \"2026-09-17T13:39:01.109176495Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m50s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000df7160\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2mtbkjvnnzmc\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0607f3e-d3c6-4002-afc3-b216aabd4af7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a0607f3e-d3c6-4002-afc3-b216aabd4af7\",\n \"name\": \"a0607f3e-d3c6-4002-afc3-b216aabd4af7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:09:52.721401557Z\",\n \"endTime\": \"2026-09-17T13:39:01.109176495Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m50s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"75524c106d9227ff82344275b36d057872cde70367aa48023ebffa47cae0a6bf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","failed_at":"e2e","signature_id":"bb7a8e5fa783928e378dcc846c79e38f12b390e8f34e30b5e6460dbc694c3af8","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f6ade0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-txp6prl987ll\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c77e7129-6fb9-48eb-b5a7-009ab48c6967\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c77e7129-6fb9-48eb-b5a7-009ab48c6967\",\n \"name\": \"c77e7129-6fb9-48eb-b5a7-009ab48c6967\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:08:46.266706852Z\",\n \"endTime\": \"2026-09-17T13:37:50.130353042Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f6ade0\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-txp6prl987ll\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c77e7129-6fb9-48eb-b5a7-009ab48c6967\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c77e7129-6fb9-48eb-b5a7-009ab48c6967\",\n \"name\": \"c77e7129-6fb9-48eb-b5a7-009ab48c6967\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:08:46.266706852Z\",\n \"endTime\": \"2026-09-17T13:37:50.130353042Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"959fe28eb03a0843988cd02959f4542f28f5c40a3b3f9bb3b03b0d59dc344e32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","failed_at":"alert","signature_id":"777817b1e387b1f4492402d2f5429c721df7c24785215c5257845216b59cfccd","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:23:01Z\n Ended: 2026-09-17T13:48:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2652928-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssph8fr577g8dbjaqg0tloglfhip8ib-y7i4q7g4c7s7i5z\", pod=\"router-894997d67-7c5ck\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssph8fr577g8dbjaqg0tloglfhip8ib-y7i4q7g4c7s7i5z/router-894997d67-7c5ck has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T13:23:01Z\n Ended: 2026-09-17T13:48:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2652928-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspgnuu0iphnirf8pnuijesopcagti4-m2g9d8c5h8m2v1p\", pod=\"router-954856bf9-78nmw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspgnuu0iphnirf8pnuijesopcagti4-m2g9d8c5h8m2v1p/router-954856bf9-78nmw has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T13:23:01Z\n Ended: 2026-09-17T13:48:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2652928-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspgnp46398o3uuu74c13ce8b0gj7i2-v20260901\", pod=\"router-bc9c99475-44czs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspgnp46398o3uuu74c13ce8b0gj7i2-v20260901/router-bc9c99475-44czs has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T13:23:01Z\n Ended: 2026-09-17T13:46:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2652928-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspguqf18fp17m9t6tqvviau001p1ol-r7d2n7a9f0o0l3p\", pod=\"router-69689b7c68-b85dk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspguqf18fp17m9t6tqvviau001p1ol-r7d2n7a9f0o0l3p/router-69689b7c68-b85dk has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-17T13:23:01Z\n Ended: 2026-09-17T13:48:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2652928-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspgqdlc1rn1r1tm8lpqprm3tucttvj-private-ingress\", pod=\"router-666f48c4b5-ksh9p\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspgqdlc1rn1r1tm8lpqprm3tucttvj-private-ingress/router-666f48c4b5-ksh9p has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-17T13:23:01Z\n Ended: 2026-09-17T13:48:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2652928-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspha5kdni12tmq452o97bfrlna4nmg-private-kas\", pod=\"router-5cff7b7494-tl542\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspha5kdni12tmq452o97bfrlna4nmg-private-kas/router-5cff7b7494-tl542 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"ab13dd2dc994fdeaef3ce91af17bd713f99cd64435356c7778447161cc1585c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","failed_at":"e2e","signature_id":"401c9dc75b4b5ec84bc7eacc4dd5a3649404e5b9bb0cad164dff2ac11981d6c4","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db2f00\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jtbcrf6fbph7\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b5818a0a-e8dd-420e-9ceb-3bf8f83ab02f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b5818a0a-e8dd-420e-9ceb-3bf8f83ab02f\",\n \"name\": \"b5818a0a-e8dd-420e-9ceb-3bf8f83ab02f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:08:56.52781692Z\",\n \"endTime\": \"2026-09-17T13:38:00.207851281Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000db2f00\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-jtbcrf6fbph7\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b5818a0a-e8dd-420e-9ceb-3bf8f83ab02f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b5818a0a-e8dd-420e-9ceb-3bf8f83ab02f\",\n \"name\": \"b5818a0a-e8dd-420e-9ceb-3bf8f83ab02f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:08:56.52781692Z\",\n \"endTime\": \"2026-09-17T13:38:00.207851281Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m7s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c26bb3fc0ee7fc480ec8d54ae4a287c4be88bdb42a056a2af7f079d26c3a37f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","failed_at":"e2e","signature_id":"236cb0e5207b1956ffac77c17fe63e39c6df7e6a37c02c56004d161df8a008ca","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-b96ppk6wk4k9/cluster-candidate-4-20-7tcz5g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00140a620\u003e: \n failed to create HCP cluster cluster-candidate-4-20-7tcz5g: failed waiting for cluster=\"cluster-candidate-4-20-7tcz5g\" in resourcegroup=\"rg-candidate-4-20-b96ppk6wk4k9\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc41842e-dec4-4955-9bfc-4e2ff8b8270f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc41842e-dec4-4955-9bfc-4e2ff8b8270f\",\n \"name\": \"bc41842e-dec4-4955-9bfc-4e2ff8b8270f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:08:46.930094659Z\",\n \"endTime\": \"2026-09-17T13:37:50.186246427Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 22m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-b96ppk6wk4k9/cluster-candidate-4-20-7tcz5g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00140a620\u003e: \n failed to create HCP cluster cluster-candidate-4-20-7tcz5g: failed waiting for cluster=\"cluster-candidate-4-20-7tcz5g\" in resourcegroup=\"rg-candidate-4-20-b96ppk6wk4k9\" to finish creating: GET https://rp.j2652928.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc41842e-dec4-4955-9bfc-4e2ff8b8270f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc41842e-dec4-4955-9bfc-4e2ff8b8270f\",\n \"name\": \"bc41842e-dec4-4955-9bfc-4e2ff8b8270f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:08:46.930094659Z\",\n \"endTime\": \"2026-09-17T13:37:50.186246427Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 22m48s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6994/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100557383175507968","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6994,"pr_state":"open","pr_sha":"87fef875b667861b4b7e12931a058e38bb650e05","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T12:07:52Z","started_at":"2026-09-17T12:07:52Z","completed_at":"2026-09-17T14:00:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:15.055762543Z","timing_metadata_checked_at":"2026-09-18T15:41:15.055762543Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:15.062940169Z","region_metadata_checked_at":"2026-09-18T15:41:15.062940169Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"06c4649a3bad4a3ae5840db0585d4212a3140c134e40f843d0da39688272bfc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6994/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100557383175507968","occurred_at":"2026-09-17T12:07:52Z","failed_at":"e2e","signature_id":"ee498f50c2b2ac982c301f4ac0cae3713723036b9fbee8e63fbb794dec859320","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-srrxr479d7ch/cluster-candidate-4-20-rh4pb8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e3940\u003e: \n failed to create HCP cluster cluster-candidate-4-20-rh4pb8: failed waiting for cluster=\"cluster-candidate-4-20-rh4pb8\" in resourcegroup=\"rg-candidate-4-20-srrxr479d7ch\" to finish creating: GET https://rp.j5507968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38bd640e-c367-4ef7-a27c-0a98287d687e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38bd640e-c367-4ef7-a27c-0a98287d687e\",\n \"name\": \"38bd640e-c367-4ef7-a27c-0a98287d687e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:10:10.863639821Z\",\n \"endTime\": \"2026-09-17T13:39:18.855692604Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-20-srrxr479d7ch/cluster-candidate-4-20-rh4pb8 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e3940\u003e: \n failed to create HCP cluster cluster-candidate-4-20-rh4pb8: failed waiting for cluster=\"cluster-candidate-4-20-rh4pb8\" in resourcegroup=\"rg-candidate-4-20-srrxr479d7ch\" to finish creating: GET https://rp.j5507968.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38bd640e-c367-4ef7-a27c-0a98287d687e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/38bd640e-c367-4ef7-a27c-0a98287d687e\",\n \"name\": \"38bd640e-c367-4ef7-a27c-0a98287d687e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:10:10.863639821Z\",\n \"endTime\": \"2026-09-17T13:39:18.855692604Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"4db07cee31621d1817087a640adb7c02e4b481af92e820b63b6913be6dacbf14","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6994/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100557383175507968","occurred_at":"2026-09-17T12:07:52Z","failed_at":"alert","signature_id":"b66eb0c3d6894149e8e5bee7c3f68c812916f17da4df7663f06be67bb5aedc35","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:24:17Z\n Ended: 2026-09-17T13:48:17Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5507968-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssphd4ggk7ujcv9c3mpfi14jorh70dh-a6v8w8w3u5q9y3a\", pod=\"router-7b5888dd5c-hmrr8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssphd4ggk7ujcv9c3mpfi14jorh70dh-a6v8w8w3u5q9y3a/router-7b5888dd5c-hmrr8 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6672,"pr_state":"closed","pr_sha":"a5dd04d6a33bf632a71bc38142e6589b697bb639","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T12:05:14Z","started_at":"2026-09-17T12:05:14Z","completed_at":"2026-09-17T14:05:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:11.496563256Z","timing_metadata_checked_at":"2026-09-18T15:40:11.496563256Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:11.50028447Z","region_metadata_checked_at":"2026-09-18T15:40:11.50028447Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"5b920d3c9e05d7f03a4564850d6136703af38c79bbd30671636fc97012bcf109","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","failed_at":"e2e","signature_id":"7be65470254fc4c585209a09735c416c75334c6ead21b5d5328120538ae66b13","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:85]: Timed out after 900.000s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:83 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:85]: Timed out after 900.000s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:83 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"35fcc68c27835aae14daafdef12fc813fb2e38cc3e35d2f974eb6cc333b3ce7e","failure_pattern":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane","search_query":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane"}},{"row_id":"647f63f04f9ff31cc064337ec0957d4312cd3c39b956332be220e9313b981936","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","failed_at":"e2e","signature_id":"99c5a5b19f403073edd6e8406f2272a3fda7ff44e9a289f5c999503a39917368","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000aac060\u003e: \n failed to get kusto cluster name from config: configuration: key kusto not found\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc000aac060\u003e: \n failed to get kusto cluster name from config: configuration: key kusto not found\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a","failure_pattern":"failed to get kusto cluster \u003ckusto-cluster\u003e from config: configuration: key kusto not found","search_query":"failed to get kusto cluster name from config: configuration: key kusto not found"}},{"row_id":"ce578aeb1a588ed0c7a565011bb77cd68a8af2cbbc1209546c36c565ed2dd67a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","failed_at":"alert","signature_id":"8d0248f7d9c1bf72afe316ba52f0dec7aa9b6c443c3506cf8e8d1a57be0d1c1b","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 36 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:08:59Z\n Ended: 2026-09-17T13:33:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcb7sqsldk7bi5dnp0oa4r7mbmdgt-k9m1g1q6v1l7j4e\", pod=\"kube-state-metrics-hcp-8654bffc85-sw4zr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcb7sqsldk7bi5dnp0oa4r7mbmdgt-k9m1g1q6v1l7j4e/kube-state-metrics-hcp-8654bffc85-sw4zr has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T13:08:59Z\n Ended: 2026-09-17T13:29:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspc83v54i8b9ktds0mno1jqu66puk8-a3i1c3s3v5b7y2b\", pod=\"kube-state-metrics-hcp-8654bffc85-vj9bl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspc83v54i8b9ktds0mno1jqu66puk8-a3i1c3s3v5b7y2b/kube-state-metrics-hcp-8654bffc85-vj9bl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T13:08:59Z\n Ended: 2026-09-17T13:28:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcab6g26b61ugvq1qa8er5s26ukr5-n8g9h1m4z5h0y5j\", pod=\"kube-state-metrics-hcp-8654bffc85-84h9q\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcab6g26b61ugvq1qa8er5s26ukr5-n8g9h1m4z5h0y5j/kube-state-metrics-hcp-8654bffc85-84h9q has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T13:08:59Z\n Ended: 2026-09-17T13:34:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcd1ibd9bsldcjivfv1pv891lupgb-g3q4b0s4i5m4w8a\", pod=\"kube-state-metrics-hcp-8654bffc85-74h7f\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcd1ibd9bsldcjivfv1pv891lupgb-g3q4b0s4i5m4w8a/kube-state-metrics-hcp-8654bffc85-74h7f has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-17T13:08:59Z\n Ended: 2026-09-17T13:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcd5pitmepf4tthfr105bf0u6qjkj-l6d4u3q2l8k1p4w\", pod=\"kube-state-metrics-hcp-8654bffc85-gl5hc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcd5pitmepf4tthfr105bf0u6qjkj-l6d4u3q2l8k1p4w/kube-state-metrics-hcp-8654bffc85-gl5hc has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-17T13:10:00Z\n Ended: 2026-09-17T13:49:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspch2tl2fng9je6t8uouikhvml9pr5-y6x5d3t0a0o9h1v\", pod=\"kube-state-metrics-hcp-8654bffc85-w544w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspch2tl2fng9je6t8uouikhvml9pr5-y6x5d3t0a0o9h1v/kube-state-metrics-hcp-8654bffc85-w544w has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-17T13:10:00Z\n Ended: 2026-09-17T13:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspca9memkr10lu9ab1oaph4b2tmsjc-v3o4o5x5g9j2j1f\", pod=\"kube-state-metrics-hcp-8654bffc85-qxmgb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspca9memkr10lu9ab1oaph4b2tmsjc-v3o4o5x5g9j2j1f/kube-state-metrics-hcp-8654bffc85-qxmgb has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:28:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcd48a8v6hh0ifl98g9db1ua89n14-sre-hcp-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-fjj8q\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcd48a8v6hh0ifl98g9db1ua89n14-sre-hcp-cluster/kube-state-metrics-hcp-8654bffc85-fjj8q has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:37:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcfk8ph0gcjmvrcofvhgjqg35bstr-q8r0w4i7h4v4z5u\", pod=\"kube-state-metrics-hcp-8654bffc85-2c7r8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcfk8ph0gcjmvrcofvhgjqg35bstr-q8r0w4i7h4v4z5u/kube-state-metrics-hcp-8654bffc85-2c7r8 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:18:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcf7ksaqkttkkr2cgq4nadfq3tol1-a5h0f7c3a0r6a1o\", pod=\"kube-state-metrics-hcp-8654bffc85-lrbq6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcf7ksaqkttkkr2cgq4nadfq3tol1-a5h0f7c3a0r6a1o/kube-state-metrics-hcp-8654bffc85-lrbq6 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:39:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcaho1pdo9bnsb9380imd1q0cgsht-n6s0o9o5z7w5l1x\", pod=\"kube-state-metrics-hcp-8654bffc85-68k4l\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcaho1pdo9bnsb9380imd1q0cgsht-n6s0o9o5z7w5l1x/kube-state-metrics-hcp-8654bffc85-68k4l has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:48:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcm0fc6pbiglveskvt7o8lqpfe1ni-q7t6j5c5r6b6e3x\", pod=\"kube-state-metrics-hcp-8654bffc85-25xjz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcm0fc6pbiglveskvt7o8lqpfe1ni-q7t6j5c5r6b6e3x/kube-state-metrics-hcp-8654bffc85-25xjz has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:50:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspceeeteqt56ehbc42dgm3btdci716-x9h9l3q6w6d2q0m\", pod=\"kube-state-metrics-hcp-8654bffc85-v45vb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspceeeteqt56ehbc42dgm3btdci716-x9h9l3q6w6d2q0m/kube-state-metrics-hcp-8654bffc85-v45vb has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:47:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcem0sa8hpfqifjev3vv7bold7e92-a5w3q3w8f1o3x9b\", pod=\"kube-state-metrics-hcp-8654bffc85-dxx9w\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcem0sa8hpfqifjev3vv7bold7e92-a5w3q3w8f1o3x9b/kube-state-metrics-hcp-8654bffc85-dxx9w has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:34:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcge206hudf057d4rapvtvmgma2mo-oidc-wi-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-hckmt\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcge206hudf057d4rapvtvmgma2mo-oidc-wi-cluster/kube-state-metrics-hcp-8654bffc85-hckmt has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:31:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspccnril3e7pdaev0lngoai5csi2ku-ea-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-pbjqj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspccnril3e7pdaev0lngoai5csi2ku-ea-cluster/kube-state-metrics-hcp-8654bffc85-pbjqj has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:19:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcm4o5ijtniiue45eh50id70dfp6a-f9t9u0r3u1b9n0r\", pod=\"kube-state-metrics-hcp-8654bffc85-w8w27\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcm4o5ijtniiue45eh50id70dfp6a-f9t9u0r3u1b9n0r/kube-state-metrics-hcp-8654bffc85-w8w27 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:18:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspca8036n0fivuel444dlkbothfifs-basic-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-47zwq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspca8036n0fivuel444dlkbothfifs-basic-cluster/kube-state-metrics-hcp-8654bffc85-47zwq has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:37:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcabjfnqjhlbhj0oghprl8hrp1oio-c6z8w3p7h3e9r9m\", pod=\"kube-state-metrics-hcp-8654bffc85-pwt7b\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcabjfnqjhlbhj0oghprl8hrp1oio-c6z8w3p7h3e9r9m/kube-state-metrics-hcp-8654bffc85-pwt7b has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-17T13:10:56Z\n Ended: 2026-09-17T13:37:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspccnmknrpuh0747ob6ahb5jenvu9k-r4g1h9p4c5j0q6i\", pod=\"kube-state-metrics-hcp-8654bffc85-5rw4j\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspccnmknrpuh0747ob6ahb5jenvu9k-r4g1h9p4c5j0q6i/kube-state-metrics-hcp-8654bffc85-5rw4j has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-17T13:10:57Z\n Ended: 2026-09-17T13:19:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcmkp48ol9sgagi11b09j67pb9ul7-l0x7x1b4p1n8t9g\", pod=\"kube-state-metrics-hcp-8654bffc85-nwbg8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcmkp48ol9sgagi11b09j67pb9ul7-l0x7x1b4p1n8t9g/kube-state-metrics-hcp-8654bffc85-nwbg8 has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-17T13:10:57Z\n Ended: 2026-09-17T13:26:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspccfhe1egj635kk9b5u2sg6m1iqmv-ea-listwgh4rg\", pod=\"kube-state-metrics-hcp-8654bffc85-mldrv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspccfhe1egj635kk9b5u2sg6m1iqmv-ea-listwgh4rg/kube-state-metrics-hcp-8654bffc85-mldrv has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-17T13:10:57Z\n Ended: 2026-09-17T13:19:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcmrjnf8qshgsel2n8pu5vcrqo8ib-p2l2b5d6x5j0p7f\", pod=\"kube-state-metrics-hcp-8654bffc85-29p94\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcmrjnf8qshgsel2n8pu5vcrqo8ib-p2l2b5d6x5j0p7f/kube-state-metrics-hcp-8654bffc85-29p94 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-17T13:10:57Z\n Ended: 2026-09-17T13:41:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcgs41mednuce8v8juoarvre0o7f1-y1p7a8c3l5z0p6s\", pod=\"kube-state-metrics-hcp-8654bffc85-fdbbb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcgs41mednuce8v8juoarvre0o7f1-y1p7a8c3l5z0p6s/kube-state-metrics-hcp-8654bffc85-fdbbb has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:46:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcee5col9vs3ec1hc99u0e0cg6rod-l5t0c4o8b7t1m6k\", pod=\"kube-state-metrics-hcp-8654bffc85-cht7p\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcee5col9vs3ec1hc99u0e0cg6rod-l5t0c4o8b7t1m6k/kube-state-metrics-hcp-8654bffc85-cht7p has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:42:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspdfg78lat8k87fj5b9k4jmbs8qicd-x6v7h0p9g2z4c9r\", pod=\"kube-state-metrics-hcp-8654bffc85-sg492\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspdfg78lat8k87fj5b9k4jmbs8qicd-x6v7h0p9g2z4c9r/kube-state-metrics-hcp-8654bffc85-sg492 has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:49:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcen9oo9lcu99o5iueg61nfrsive0-v5c8o3g3i0h7h3b\", pod=\"kube-state-metrics-hcp-8654bffc85-rrg4m\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcen9oo9lcu99o5iueg61nfrsive0-v5c8o3g3i0h7h3b/kube-state-metrics-hcp-8654bffc85-rrg4m has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:34:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspccsnbr6b8rqfb4lsjsi86jn0doel-cilium-cl\", pod=\"kube-state-metrics-hcp-8654bffc85-5twhb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspccsnbr6b8rqfb4lsjsi86jn0doel-cilium-cl/kube-state-metrics-hcp-8654bffc85-5twhb has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:41:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspccdb2mcj9vfuqhqc0eshri0b51gk-c1n4i1l5h3v3w5y\", pod=\"kube-state-metrics-hcp-8654bffc85-8tkwq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspccdb2mcj9vfuqhqc0eshri0b51gk-c1n4i1l5h3v3w5y/kube-state-metrics-hcp-8654bffc85-8tkwq has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:48:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcm0k5ga93uqot5b5roh4hu0pno7l-b3e0v4h4q2l7f4x\", pod=\"kube-state-metrics-hcp-8654bffc85-mbg2n\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcm0k5ga93uqot5b5roh4hu0pno7l-b3e0v4h4q2l7f4x/kube-state-metrics-hcp-8654bffc85-mbg2n has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:39:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcf3m7huj2ui7lb82bh163g9kpqpp-s6b5r7u6f2z2f3z\", pod=\"kube-state-metrics-hcp-8654bffc85-twgnb\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcf3m7huj2ui7lb82bh163g9kpqpp-s6b5r7u6f2z2f3z/kube-state-metrics-hcp-8654bffc85-twgnb has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:25:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspccu0f6eg3g5158smlj1ia6m6j77b-t1u8x2j7n4m3n7v\", pod=\"kube-state-metrics-hcp-8654bffc85-smqcz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspccu0f6eg3g5158smlj1ia6m6j77b-t1u8x2j7n4m3n7v/kube-state-metrics-hcp-8654bffc85-smqcz has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:29:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspccsal4ptn0kdvo8ga8voqcluuqn5-g1i8q2w1x1z3q4o\", pod=\"kube-state-metrics-hcp-8654bffc85-7pnz5\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspccsal4ptn0kdvo8ga8voqcluuqn5-g1i8q2w1x1z3q4o/kube-state-metrics-hcp-8654bffc85-7pnz5 has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:41:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcekf3ud0i2iqs827rrhieggpvun6-t3z2j3v2v1y5a6t\", pod=\"kube-state-metrics-hcp-8654bffc85-df2sj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcekf3ud0i2iqs827rrhieggpvun6-t3z2j3v2v1y5a6t/kube-state-metrics-hcp-8654bffc85-df2sj has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:31:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspclmnibunjprapdm7j3gamrhiduda-d2i3i0b5r0s4j5j\", pod=\"kube-state-metrics-hcp-8654bffc85-prqn6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspclmnibunjprapdm7j3gamrhiduda-d2i3i0b5r0s4j5j/kube-state-metrics-hcp-8654bffc85-prqn6 has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-09-17T13:12:00Z\n Ended: 2026-09-17T13:36:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5128576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspcfko5q3bri1mmij2t32jidfdi6ri-t5a3k3q0r9w8x3a\", pod=\"kube-state-metrics-hcp-8654bffc85-bpdfw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspcfko5q3bri1mmij2t32jidfdi6ri-t5a3k3q0r9w8x3a/kube-state-metrics-hcp-8654bffc85-bpdfw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["35fcc68c27835aae14daafdef12fc813fb2e38cc3e35d2f974eb6cc333b3ce7e","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6993,"pr_state":"open","pr_sha":"451b61ce4916b5f8bc6f4227d2a36723c517d17e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T11:57:31Z","started_at":"2026-09-17T11:57:31Z","completed_at":"2026-09-17T14:05:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:14.910293717Z","timing_metadata_checked_at":"2026-09-18T15:41:14.910293717Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:14.913561429Z","region_metadata_checked_at":"2026-09-18T15:41:14.913561429Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"02eb817180e6733d4519d3a70ad8eccf75bc67f626fb74a6c6101124561c0c35","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"26a26ee01c1fe63e9854b87ab08eead6a6fc7c9019b032530af1bdbbab01d85b","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001790120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8928t8c2w987\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n \"name\": \"7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:04:21.545838494Z\",\n \"endTime\": \"2026-09-17T13:28:30.638150402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp95g2hh8pms8h7312ekq6dl4tr17v still exists (deletion dispatched at 2026-09-17T13:04:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001790120\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8928t8c2w987\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n \"name\": \"7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:04:21.545838494Z\",\n \"endTime\": \"2026-09-17T13:28:30.638150402Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp95g2hh8pms8h7312ekq6dl4tr17v still exists (deletion dispatched at 2026-09-17T13:04:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0937810175b5f1ac2a40954c1f7c4112d4a2ab924141de1b3445aa92522a7936","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"36c7d0ac2608fdd9b180eb76cfcef49dd063a5ddca7a5e1dd0eb82655e85e2bc","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011b4600\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-8sx86dmkm86k\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13066ffb-8ec0-49b0-91b1-8f95cc778712\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13066ffb-8ec0-49b0-91b1-8f95cc778712\",\n \"name\": \"13066ffb-8ec0-49b0-91b1-8f95cc778712\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:13:55.616199714Z\",\n \"endTime\": \"2026-09-17T13:38:00.670848783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp9797225e4pc4eg93kcbplgmr3noo still exists (deletion dispatched at 2026-09-17T13:14:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0011b4600\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-8sx86dmkm86k\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13066ffb-8ec0-49b0-91b1-8f95cc778712\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13066ffb-8ec0-49b0-91b1-8f95cc778712\",\n \"name\": \"13066ffb-8ec0-49b0-91b1-8f95cc778712\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:13:55.616199714Z\",\n \"endTime\": \"2026-09-17T13:38:00.670848783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp9797225e4pc4eg93kcbplgmr3noo still exists (deletion dispatched at 2026-09-17T13:14:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2a95b41518bd39c9fe6e66edaa913990dcf63649ad4115d58692d267b526f649","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"54d2d9e8120866d9194ddacd546d77becc75e8a251ac250bfb9fb8db3e040d85","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0027021e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-fvq6xfsmv5rt\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06520f2e-e19c-4091-a068-c1fb58d73111\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06520f2e-e19c-4091-a068-c1fb58d73111\",\n \"name\": \"06520f2e-e19c-4091-a068-c1fb58d73111\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:03:13.698113353Z\",\n \"endTime\": \"2026-09-17T13:27:20.680352254Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7jvtggrghuov8k46cvi5s0t83kgc still exists (deletion dispatched at 2026-09-17T13:03:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0027021e0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-fvq6xfsmv5rt\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06520f2e-e19c-4091-a068-c1fb58d73111\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06520f2e-e19c-4091-a068-c1fb58d73111\",\n \"name\": \"06520f2e-e19c-4091-a068-c1fb58d73111\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:03:13.698113353Z\",\n \"endTime\": \"2026-09-17T13:27:20.680352254Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7jvtggrghuov8k46cvi5s0t83kgc still exists (deletion dispatched at 2026-09-17T13:03:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"33a216ec5741300e3fb38ec3f551bc520c91a64a4a21abfdf59004373fd19190","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"51e487e0e9fc1b87185142de83749a342d6962b63a6106533902cf6918e54d5e","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000012e40\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-67hdvl9w4j6q\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4599e3d2-e4d0-4e28-8c3a-e6618ada2d97\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4599e3d2-e4d0-4e28-8c3a-e6618ada2d97\",\n \"name\": \"4599e3d2-e4d0-4e28-8c3a-e6618ada2d97\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:15.532571894Z\",\n \"endTime\": \"2026-09-17T13:35:20.695248589Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp8bghu2kio5ia0opa50hpqiea2oa0 still exists (deletion dispatched at 2026-09-17T13:11:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000012e40\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-67hdvl9w4j6q\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4599e3d2-e4d0-4e28-8c3a-e6618ada2d97\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4599e3d2-e4d0-4e28-8c3a-e6618ada2d97\",\n \"name\": \"4599e3d2-e4d0-4e28-8c3a-e6618ada2d97\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:15.532571894Z\",\n \"endTime\": \"2026-09-17T13:35:20.695248589Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp8bghu2kio5ia0opa50hpqiea2oa0 still exists (deletion dispatched at 2026-09-17T13:11:20Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"480eda92cfe4927341b2effc7996ebccb853e73174b185036bfae9e77929db1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"1200ec6f2e462f6a4057f0b6736f389e0eb4918e85be52611ba1988ed4a6b80f","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001226768\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jwdh6h\" in resourcegroup=\"rg-candidate-4-22-9rtkhgwpwkdp\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\",\n \"name\": \"497b563d-6a90-4107-8ea2-18e801b64cd8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:09:44.365139487Z\",\n \"endTime\": \"2026-09-17T13:33:50.717397424Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7gelb6kbkk61h7dh9rg6bgds1mn2 still exists (deletion dispatched at 2026-09-17T13:09:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001226768\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jwdh6h\" in resourcegroup=\"rg-candidate-4-22-9rtkhgwpwkdp\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\",\n \"name\": \"497b563d-6a90-4107-8ea2-18e801b64cd8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:09:44.365139487Z\",\n \"endTime\": \"2026-09-17T13:33:50.717397424Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7gelb6kbkk61h7dh9rg6bgds1mn2 still exists (deletion dispatched at 2026-09-17T13:09:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"59971244d2585c68ee359d6b4a1291963b7b0d7248564651a0cd45431a3facf4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"d5580f5abed7b32fb1241b3c5a25421aa30df617d15c7a847acda8ee9363e8f9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00123e3a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-qzhgv4\" in resourcegroup=\"rg-candidate-4-20-w4h7vw7ct2fr\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6fd48b8c-0627-4555-8c0e-d37ed82eb815\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6fd48b8c-0627-4555-8c0e-d37ed82eb815\",\n \"name\": \"6fd48b8c-0627-4555-8c0e-d37ed82eb815\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:04:19.914202757Z\",\n \"endTime\": \"2026-09-17T13:28:20.692107951Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7811gchpng82erpcrdlvqh636u2b still exists (deletion dispatched at 2026-09-17T13:04:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00123e3a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-qzhgv4\" in resourcegroup=\"rg-candidate-4-20-w4h7vw7ct2fr\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6fd48b8c-0627-4555-8c0e-d37ed82eb815\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6fd48b8c-0627-4555-8c0e-d37ed82eb815\",\n \"name\": \"6fd48b8c-0627-4555-8c0e-d37ed82eb815\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:04:19.914202757Z\",\n \"endTime\": \"2026-09-17T13:28:20.692107951Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7811gchpng82erpcrdlvqh636u2b still exists (deletion dispatched at 2026-09-17T13:04:21Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"5e5e745da7043a8a128ae316331c23a80e9751d0a316d3f6bf972ca38e2e0e8a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"83cf0cf00f201ed62094249bde1cad4326fd870d9fd95f573466f70f995ca9f3","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014980d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-j826q67vvj52\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/280170cf-c530-4d72-9037-9f04f2905802\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/280170cf-c530-4d72-9037-9f04f2905802\",\n \"name\": \"280170cf-c530-4d72-9037-9f04f2905802\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:00:52.342655089Z\",\n \"endTime\": \"2026-09-17T13:25:00.74293906Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp73alrk9qecl63aq268esm2m41s6e still exists (deletion dispatched at 2026-09-17T13:00:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0014980d8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"delayed-rbac-cluster\" in resourcegroup=\"delayed-rbac-j826q67vvj52\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/280170cf-c530-4d72-9037-9f04f2905802\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/280170cf-c530-4d72-9037-9f04f2905802\",\n \"name\": \"280170cf-c530-4d72-9037-9f04f2905802\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:00:52.342655089Z\",\n \"endTime\": \"2026-09-17T13:25:00.74293906Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp73alrk9qecl63aq268esm2m41s6e still exists (deletion dispatched at 2026-09-17T13:00:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b1ea7a5dad2619c4f7d18e07de1a4f1ee3128c0e94c4ef3b73c5051aa38a3f44","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"a7c143432b9aa1db66f17c395896aa0a31866970f3d2152b89df2ba59a93d938","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00102e210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-r9k66x\" in resourcegroup=\"rg-np-noedge-r9k66x-b6q65mpdbrdl\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4993b33a-2f46-49c4-9ddc-e8dfd573aadd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4993b33a-2f46-49c4-9ddc-e8dfd573aadd\",\n \"name\": \"4993b33a-2f46-49c4-9ddc-e8dfd573aadd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:07:12.283709141Z\",\n \"endTime\": \"2026-09-17T13:31:20.695028077Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7ju1gnqcl4frelh23hftlr0i6s8h still exists (deletion dispatched at 2026-09-17T13:07:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00102e210\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-noedge-r9k66x\" in resourcegroup=\"rg-np-noedge-r9k66x-b6q65mpdbrdl\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4993b33a-2f46-49c4-9ddc-e8dfd573aadd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4993b33a-2f46-49c4-9ddc-e8dfd573aadd\",\n \"name\": \"4993b33a-2f46-49c4-9ddc-e8dfd573aadd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:07:12.283709141Z\",\n \"endTime\": \"2026-09-17T13:31:20.695028077Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7ju1gnqcl4frelh23hftlr0i6s8h still exists (deletion dispatched at 2026-09-17T13:07:13Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b63c0476eff1c08ddf37bb2b3e03658d17b349ce32423c5055e75ab12291d651","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"f1d1ec88637d584406805321e304585733a1d0d5f1dc372e90627e8f47025e62","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012b0228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-q27rc669zqsh\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fd21058-2067-490c-af21-4f8dcf7eaa2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fd21058-2067-490c-af21-4f8dcf7eaa2d\",\n \"name\": \"7fd21058-2067-490c-af21-4f8dcf7eaa2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:18:15.92690988Z\",\n \"endTime\": \"2026-09-17T13:42:20.66158476Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp9ceiqjg99tofkj78h4d9u0bhnv3l still exists (deletion dispatched at 2026-09-17T13:18:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012b0228\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-q27rc669zqsh\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fd21058-2067-490c-af21-4f8dcf7eaa2d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7fd21058-2067-490c-af21-4f8dcf7eaa2d\",\n \"name\": \"7fd21058-2067-490c-af21-4f8dcf7eaa2d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:18:15.92690988Z\",\n \"endTime\": \"2026-09-17T13:42:20.66158476Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp9ceiqjg99tofkj78h4d9u0bhnv3l still exists (deletion dispatched at 2026-09-17T13:18:17Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c78c756a30e73904f8dbe71584cff9146bf6e018a4b694f3b9ebb527b013235c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"f1c4779490fb4bdafbfc35cb730136fb51e8d2baae0539cf8ed6ed1617f9f5b5","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ddc1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-n5x5stk2jr2b\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/581beceb-4aec-4acd-855a-b3a0fe1b475e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/581beceb-4aec-4acd-855a-b3a0fe1b475e\",\n \"name\": \"581beceb-4aec-4acd-855a-b3a0fe1b475e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:03:11.454975969Z\",\n \"endTime\": \"2026-09-17T13:27:20.747941158Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7847nq93qpgrlmug2nh4ioibnstg still exists (deletion dispatched at 2026-09-17T13:03:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001ddc1b0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-n5x5stk2jr2b\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/581beceb-4aec-4acd-855a-b3a0fe1b475e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/581beceb-4aec-4acd-855a-b3a0fe1b475e\",\n \"name\": \"581beceb-4aec-4acd-855a-b3a0fe1b475e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:03:11.454975969Z\",\n \"endTime\": \"2026-09-17T13:27:20.747941158Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7847nq93qpgrlmug2nh4ioibnstg still exists (deletion dispatched at 2026-09-17T13:03:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"de87c0b58d9e69ba93840296142ebbe9d6739c488487c939d76db3d8601370fb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"397c03aebb8f9974cabd4c075afe35800b5602c55edd622b828e4ee997bc99ca","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001002300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-rs72gd\" in resourcegroup=\"rg-np-downgrade-rs72gd-h2sg5z57shg4\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc3d9638-28ce-4c55-a9f7-10d6903cf30f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc3d9638-28ce-4c55-a9f7-10d6903cf30f\",\n \"name\": \"cc3d9638-28ce-4c55-a9f7-10d6903cf30f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:18:23.650124176Z\",\n \"endTime\": \"2026-09-17T13:42:30.681443783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp9bc7t4br61qj7lck2cv5908qu6rv still exists (deletion dispatched at 2026-09-17T13:18:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001002300\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-rs72gd\" in resourcegroup=\"rg-np-downgrade-rs72gd-h2sg5z57shg4\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc3d9638-28ce-4c55-a9f7-10d6903cf30f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc3d9638-28ce-4c55-a9f7-10d6903cf30f\",\n \"name\": \"cc3d9638-28ce-4c55-a9f7-10d6903cf30f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:18:23.650124176Z\",\n \"endTime\": \"2026-09-17T13:42:30.681443783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp9bc7t4br61qj7lck2cv5908qu6rv still exists (deletion dispatched at 2026-09-17T13:18:25Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e1cdf9776aad875bf9024f0f9ac98a85ff79407f123cc38c76e157f8d6630c98","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","failed_at":"e2e","signature_id":"e37087f31e7094fe45c7ae2868faf05fde32666577b47902fe94fbe270287c3b","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a16108\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001a16108\u003e: \n failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"open","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T11:57:12Z","started_at":"2026-09-17T11:57:12Z","completed_at":"2026-09-17T14:01:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:21.078961029Z","timing_metadata_checked_at":"2026-09-18T15:40:21.078961029Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:21.08203744Z","region_metadata_checked_at":"2026-09-18T15:40:21.08203744Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"23a1c3f44d09de8562f016aa8eb39ae0e8b2c4afceced35a4e675a5055de46f0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","failed_at":"e2e","signature_id":"f4470bf38da6c976133b96af81d9ecd329780f2aa1f6bfa931cdb8bcafadf34f","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8c940\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rsjc9lcm2kdb\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/08ee0aa6-f26c-4889-ab77-6ed9a097e06b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/08ee0aa6-f26c-4889-ab77-6ed9a097e06b\",\n \"name\": \"08ee0aa6-f26c-4889-ab77-6ed9a097e06b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T12:56:47.419533085Z\",\n \"endTime\": \"2026-09-17T13:25:52.286605579Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e8c940\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-rsjc9lcm2kdb\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/08ee0aa6-f26c-4889-ab77-6ed9a097e06b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/08ee0aa6-f26c-4889-ab77-6ed9a097e06b\",\n \"name\": \"08ee0aa6-f26c-4889-ab77-6ed9a097e06b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T12:56:47.419533085Z\",\n \"endTime\": \"2026-09-17T13:25:52.286605579Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a11fe95434397c04403e7a6f8af38834810fe3b8a30a087d61bd94c5043e3cd3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","failed_at":"e2e","signature_id":"7dda8cc2984aa1b8ad7daf00d042d0064ecf3188c6f0d908489ee4571d215e40","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0009e8ec0\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n \"name\": \"f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:12:19.756903274Z\",\n \"endTime\": \"2026-09-17T13:12:54.770011764Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0009e8ec0\u003e: \n failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n \"name\": \"f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:12:19.756903274Z\",\n \"endTime\": \"2026-09-17T13:12:54.770011764Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp"}},{"row_id":"a59c84e0295f2fd70a5ea8f632ad76479ca9ef33de05213545be773e47fbf16c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","failed_at":"alert","signature_id":"6db51492a3864e98312b98dbacda80d9a85398c3e3f34314664719d56a168f65","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T13:42:12Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j4979456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4979456/providers/microsoft.monitor/accounts/services-j4979456\", namespace=\"arobit\", pod=\"arobit-forwarder-k26lv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9061149b-822a-4131-9e5f-b554ee1870de\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-k26lv on cluster ci01-j4979456-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T13:42:12Z\n Severity: Sev3\n Labels: alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j4979456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4979456/providers/microsoft.monitor/accounts/services-j4979456\", namespace=\"arobit\", pod=\"arobit-forwarder-k26lv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9061149b-822a-4131-9e5f-b554ee1870de\"\n Description: Container fluentbit in pod arobit/arobit-forwarder-k26lv on cluster ci01-j4979456-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired"}},{"row_id":"f97bba0ce0ea3eae7e16de2acde0625bc733206d623833f0edb2ace26de88f57","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","failed_at":"e2e","signature_id":"d54234423202acb80a8cc7b79b53caff0b960878b2770d3f2b50c496f32d7589","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-qhppngpsw5vq/cluster-candidate-4-22-7j8cfp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bf4500\u003e: \n failed to create HCP cluster cluster-candidate-4-22-7j8cfp: failed waiting for cluster=\"cluster-candidate-4-22-7j8cfp\" in resourcegroup=\"rg-candidate-4-22-qhppngpsw5vq\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/793cad3e-bb75-488e-8096-19ebf4446508\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/793cad3e-bb75-488e-8096-19ebf4446508\",\n \"name\": \"793cad3e-bb75-488e-8096-19ebf4446508\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T12:56:38.840875715Z\",\n \"endTime\": \"2026-09-17T13:25:42.359594909Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m12s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-qhppngpsw5vq/cluster-candidate-4-22-7j8cfp should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bf4500\u003e: \n failed to create HCP cluster cluster-candidate-4-22-7j8cfp: failed waiting for cluster=\"cluster-candidate-4-22-7j8cfp\" in resourcegroup=\"rg-candidate-4-22-qhppngpsw5vq\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/793cad3e-bb75-488e-8096-19ebf4446508\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/793cad3e-bb75-488e-8096-19ebf4446508\",\n \"name\": \"793cad3e-bb75-488e-8096-19ebf4446508\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T12:56:38.840875715Z\",\n \"endTime\": \"2026-09-17T13:25:42.359594909Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m12s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"d433c21668cfdc66f98262f9099c8086b4397248","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T11:57:00Z","started_at":"2026-09-17T11:57:00Z","completed_at":"2026-09-17T14:01:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:20.786792978Z","timing_metadata_checked_at":"2026-09-18T15:40:20.786792978Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:20.79031789Z","region_metadata_checked_at":"2026-09-18T15:40:20.79031789Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"132fdff84a0204582157a77f3ac05a1c78fde49c1327d38c587886b7535c188e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","failed_at":"alert","signature_id":"f1e05c2e51f466f034476da1ae61bfe77799ca5b43dd98ad6f2836fda4da614d","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T14:18:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7954560-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-th462\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-th462 (namespace arobit) on cluster ci01-j7954560-svc has averaged 125.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired"}},{"row_id":"339821d59252b3dfb4d08f696ff1bd0700ddaa22f3d97e44013061dcd9499eaa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","failed_at":"alert","signature_id":"f10ddf6a0df1f5dbb5292b3e49c11fae37f0c18b3ec87fcd7444bc0f39779449","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s) (1 unknown, 2 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:21:21Z\n Ended: 2026-09-17T13:42:26Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7954560-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspg3njcujb3v6v7plhp5kqm1vtammb-z1k3i6i3q2w6p1h\", pod=\"router-5ff4d5db54-z8hcs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspg3njcujb3v6v7plhp5kqm1vtammb-z1k3i6i3q2w6p1h/router-5ff4d5db54-z8hcs has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"36abf9f098cb3fa3f275bb994b90dce83136bf5e9d93b5b1a261dc5ca94bdf0b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","failed_at":"e2e","signature_id":"8cd2901e21933a26dee207d65687d26f76901c588c4dc077ff1cf8179eac46e9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-m7mk4qp8ml44/cluster-candidate-4-22-g8z94g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b68a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-g8z94g: failed waiting for cluster=\"cluster-candidate-4-22-g8z94g\" in resourcegroup=\"rg-candidate-4-22-m7mk4qp8ml44\" to finish creating: GET https://rp.j7954560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/452782a4-ef29-4cb2-88be-222e3c6d56bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/452782a4-ef29-4cb2-88be-222e3c6d56bb\",\n \"name\": \"452782a4-ef29-4cb2-88be-222e3c6d56bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:07:19.259933821Z\",\n \"endTime\": \"2026-09-17T13:36:28.833945428Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m12s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-m7mk4qp8ml44/cluster-candidate-4-22-g8z94g should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001b68a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-g8z94g: failed waiting for cluster=\"cluster-candidate-4-22-g8z94g\" in resourcegroup=\"rg-candidate-4-22-m7mk4qp8ml44\" to finish creating: GET https://rp.j7954560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/452782a4-ef29-4cb2-88be-222e3c6d56bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/452782a4-ef29-4cb2-88be-222e3c6d56bb\",\n \"name\": \"452782a4-ef29-4cb2-88be-222e3c6d56bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:07:19.259933821Z\",\n \"endTime\": \"2026-09-17T13:36:28.833945428Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m12s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"665c7793ae61ed2fbb05fe944bd92618fcae2e087f80752f4643d24c9d741470","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","failed_at":"e2e","signature_id":"c31f9f905858e3bef622b1331144e2d854b98bc4c9aa6c6098d97c66fd0a17ca","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-rh7dxqwsn6jn/cluster-candidate-5-0-ww9f6h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c6c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-ww9f6h: failed waiting for cluster=\"cluster-candidate-5-0-ww9f6h\" in resourcegroup=\"rg-candidate-5-0-rh7dxqwsn6jn\" to finish creating: GET https://rp.j7954560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a10021c-4ce6-4db9-a901-692ad3cb7f6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a10021c-4ce6-4db9-a901-692ad3cb7f6e\",\n \"name\": \"5a10021c-4ce6-4db9-a901-692ad3cb7f6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:07:17.85124978Z\",\n \"endTime\": \"2026-09-17T13:36:18.86048967Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 18m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-rh7dxqwsn6jn/cluster-candidate-5-0-ww9f6h should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c6c0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-ww9f6h: failed waiting for cluster=\"cluster-candidate-5-0-ww9f6h\" in resourcegroup=\"rg-candidate-5-0-rh7dxqwsn6jn\" to finish creating: GET https://rp.j7954560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a10021c-4ce6-4db9-a901-692ad3cb7f6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5a10021c-4ce6-4db9-a901-692ad3cb7f6e\",\n \"name\": \"5a10021c-4ce6-4db9-a901-692ad3cb7f6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:07:17.85124978Z\",\n \"endTime\": \"2026-09-17T13:36:18.86048967Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 18m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f9c8d7c0cdb0f7092a7a661951e5ff2e544780a6a35d8929606f42d6dd7e690a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","failed_at":"e2e","signature_id":"8063e53d23cc71e09e9af725e9b15fa0500e6877a011fce15107ebe822270942","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e1220\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-l5jw7spk7gfn\" to finish creating: GET https://rp.j7954560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/047e1067-3a68-43b0-82bf-f03b03455543\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/047e1067-3a68-43b0-82bf-f03b03455543\",\n \"name\": \"047e1067-3a68-43b0-82bf-f03b03455543\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:07:16.901724077Z\",\n \"endTime\": \"2026-09-17T13:36:18.816767345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009e1220\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-l5jw7spk7gfn\" to finish creating: GET https://rp.j7954560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/047e1067-3a68-43b0-82bf-f03b03455543\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/047e1067-3a68-43b0-82bf-f03b03455543\",\n \"name\": \"047e1067-3a68-43b0-82bf-f03b03455543\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:07:16.901724077Z\",\n \"endTime\": \"2026-09-17T13:36:18.816767345Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6974,"pr_state":"closed","pr_sha":"ad18c381b0fe07c50afb22f07e5b849b9d525738","final_merged_sha":"fdb1b526d51e98ec4ae2b5a2fc9fae30dbe31d6d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T11:56:54Z","started_at":"2026-09-17T11:56:54Z","completed_at":"2026-09-17T14:02:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:04.364107155Z","timing_metadata_checked_at":"2026-09-18T15:41:04.364107155Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:04.367294967Z","region_metadata_checked_at":"2026-09-18T15:41:04.367294967Z"},"failed_at":["e2e"],"failed_test_count":6,"occurrences":[{"row_id":"59a63ec512165d81e04a626412e35fee781bc10d3b359865e3d35c78680f2e21","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","failed_at":"e2e","signature_id":"5ffc110d49d5553b43d54e9686ac2a42d55e70924452ef37b6b6d552874166f9","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f5c1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listct75l7\" in resourcegroup=\"ea-list-rnbsndngbwcp\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc1b0f96-4766-41c8-a9e1-454bcda18f72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc1b0f96-4766-41c8-a9e1-454bcda18f72\",\n \"name\": \"cc1b0f96-4766-41c8-a9e1-454bcda18f72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:03:05.503917643Z\",\n \"endTime\": \"2026-09-17T13:27:11.118627196Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp8aks7ifc0p8r9jljrcuc8dareu6k still exists (deletion dispatched at 2026-09-17T13:03:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000f5c1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"ea-listct75l7\" in resourcegroup=\"ea-list-rnbsndngbwcp\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc1b0f96-4766-41c8-a9e1-454bcda18f72\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cc1b0f96-4766-41c8-a9e1-454bcda18f72\",\n \"name\": \"cc1b0f96-4766-41c8-a9e1-454bcda18f72\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:03:05.503917643Z\",\n \"endTime\": \"2026-09-17T13:27:11.118627196Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp8aks7ifc0p8r9jljrcuc8dareu6k still exists (deletion dispatched at 2026-09-17T13:03:06Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6182969a4f1284f93467b1830ae38c88810c7299452dc332ad13a9b8346501f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","failed_at":"e2e","signature_id":"d478c40d48df905f6ee8e97dc60c2a82d7080ab4ce2d7fdcba0c3903ce1b257b","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b7e198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-k647zxtr8gg7\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de57187c-2af7-4266-8cc5-141df40a6c1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de57187c-2af7-4266-8cc5-141df40a6c1d\",\n \"name\": \"de57187c-2af7-4266-8cc5-141df40a6c1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:03:16.299863658Z\",\n \"endTime\": \"2026-09-17T13:27:21.114151007Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp758fl4976ro8f0vfe0qd8k8vok4b still exists (deletion dispatched at 2026-09-17T13:03:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001b7e198\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-k647zxtr8gg7\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de57187c-2af7-4266-8cc5-141df40a6c1d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/de57187c-2af7-4266-8cc5-141df40a6c1d\",\n \"name\": \"de57187c-2af7-4266-8cc5-141df40a6c1d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:03:16.299863658Z\",\n \"endTime\": \"2026-09-17T13:27:21.114151007Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp758fl4976ro8f0vfe0qd8k8vok4b still exists (deletion dispatched at 2026-09-17T13:03:18Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"70ce46d7bcbf697ce9b9901859a4b4855c8f3c0756d29be9e985424c10ccdc5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","failed_at":"e2e","signature_id":"1d5a773360bc54998f805181631ae95d9d4f7e700834cf8b2a7758ff8a01a8b3","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012be390\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-szkllwrp2bf5\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff3dfcf-120e-4b41-863c-96a7d57c1c10\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff3dfcf-120e-4b41-863c-96a7d57c1c10\",\n \"name\": \"0ff3dfcf-120e-4b41-863c-96a7d57c1c10\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:18.054099455Z\",\n \"endTime\": \"2026-09-17T13:35:21.06741968Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp77l69rakntvchgj57nf2vu6tgi6d still exists (deletion dispatched at 2026-09-17T13:11:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0012be390\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"private-ingress\" in resourcegroup=\"private-ingress-szkllwrp2bf5\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff3dfcf-120e-4b41-863c-96a7d57c1c10\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0ff3dfcf-120e-4b41-863c-96a7d57c1c10\",\n \"name\": \"0ff3dfcf-120e-4b41-863c-96a7d57c1c10\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:18.054099455Z\",\n \"endTime\": \"2026-09-17T13:35:21.06741968Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp77l69rakntvchgj57nf2vu6tgi6d still exists (deletion dispatched at 2026-09-17T13:11:19Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"885f33f51537af3f6ed2445272d7f96b0a7649335146e4e9e2f98ba3e750eea6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","failed_at":"e2e","signature_id":"8f0ccca9b1ab0f62cbe8877698c2eb88ab3fdc916c573bc80063829aa9d361db","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019d00c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-bv994jltql7w\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/24536fa2-db66-4ae9-88b1-d48cfc097968\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/24536fa2-db66-4ae9-88b1-d48cfc097968\",\n \"name\": \"24536fa2-db66-4ae9-88b1-d48cfc097968\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T12:59:26.848712279Z\",\n \"endTime\": \"2026-09-17T13:23:31.008071443Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7k78d1ol49mvc6mdg70kklbeg60h still exists (deletion dispatched at 2026-09-17T12:59:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0019d00c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-bv994jltql7w\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/24536fa2-db66-4ae9-88b1-d48cfc097968\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/24536fa2-db66-4ae9-88b1-d48cfc097968\",\n \"name\": \"24536fa2-db66-4ae9-88b1-d48cfc097968\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T12:59:26.848712279Z\",\n \"endTime\": \"2026-09-17T13:23:31.008071443Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7k78d1ol49mvc6mdg70kklbeg60h still exists (deletion dispatched at 2026-09-17T12:59:32Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"aed04c8277d645cf68f54fc0ddc698f0cc510b42e3a629b1b1a3b9b9c4621184","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","failed_at":"e2e","signature_id":"0e7b7131df9efbbc641dfa4dfd266b388acf38942df6680d264c1a305036569d","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105a180\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-l59lxtj9lwhd\" to finish creating: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7095545b-bbd0-41f6-961e-6bb475b19413\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7095545b-bbd0-41f6-961e-6bb475b19413\",\n \"name\": \"7095545b-bbd0-41f6-961e-6bb475b19413\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:01:16.192258014Z\",\n \"endTime\": \"2026-09-17T13:01:58.244062799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00105a180\u003e: \n failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-l59lxtj9lwhd\" to finish creating: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7095545b-bbd0-41f6-961e-6bb475b19413\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7095545b-bbd0-41f6-961e-6bb475b19413\",\n \"name\": \"7095545b-bbd0-41f6-961e-6bb475b19413\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:01:16.192258014Z\",\n \"endTime\": \"2026-09-17T13:01:58.244062799Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b494155726928001b007409414130dfb4a50196322a7329e8735581141303b1b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","failed_at":"e2e","signature_id":"f1fb92d8ffbf4155d66435595ff699141fa92d0cac0f75c60e648373d685e73b","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001221140\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-wk6m45pxns76\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b51040b7-6bf3-4b42-9a38-9f9ed135e56a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b51040b7-6bf3-4b42-9a38-9f9ed135e56a\",\n \"name\": \"b51040b7-6bf3-4b42-9a38-9f9ed135e56a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:04:07.34911158Z\",\n \"endTime\": \"2026-09-17T13:28:11.087818002Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp79q6249e2dvjoak8t3ao7t9k8ino still exists (deletion dispatched at 2026-09-17T13:04:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001221140\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-wk6m45pxns76\" to finish deleting: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b51040b7-6bf3-4b42-9a38-9f9ed135e56a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b51040b7-6bf3-4b42-9a38-9f9ed135e56a\",\n \"name\": \"b51040b7-6bf3-4b42-9a38-9f9ed135e56a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:04:07.34911158Z\",\n \"endTime\": \"2026-09-17T13:28:11.087818002Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp79q6249e2dvjoak8t3ao7t9k8ino still exists (deletion dispatched at 2026-09-17T13:04:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T11:56:52Z","started_at":"2026-09-17T11:56:52Z","completed_at":"2026-09-17T14:02:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.830158081Z","timing_metadata_checked_at":"2026-09-18T15:40:44.830158081Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:44.834845098Z","region_metadata_checked_at":"2026-09-18T15:40:44.834845098Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"311f05ccbef479e73de033182d1ca62bb5eae6c649c053181c8028e85a34f0ff","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","failed_at":"alert","signature_id":"8c32a70da3a734350e4252477cd5e4400a822058b8ff76cfef91cedb261d1712","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:27:25Z\n Ended: 2026-09-17T13:33:25Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9627648-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9627648-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"7ade154f47fd55778423a6a75c297d731cbabc6c979299cdac2c16f33d55627d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","failed_at":"alert","signature_id":"e2639a0797c1c9a4a16eba734d19e610cbd443c3f33dd4bdd37f3d91b31ccf54","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:12:31Z\n Ended: 2026-09-17T13:37:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9627648-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sspamoc80krnrof9cfovaftgbsh4v70-p4a8m9h3e4b4i4x\", pod=\"router-c6f7b99b5-tvnph\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sspamoc80krnrof9cfovaftgbsh4v70-p4a8m9h3e4b4i4x/router-c6f7b99b5-tvnph has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"7d596bdd94972b2f521d4b184f350fecb9dc6d0ab1d427ac364cb4b6b5cd9d56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","failed_at":"e2e","signature_id":"9577b3508cc1b362f70a7592604f657ac43e72478f7b4c69c3b22190b77112b8","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-jlgdwlhnhcx7/cluster-candidate-5-0-h6gm8t should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088e480\u003e: \n failed to create HCP cluster cluster-candidate-5-0-h6gm8t: failed waiting for cluster=\"cluster-candidate-5-0-h6gm8t\" in resourcegroup=\"rg-candidate-5-0-jlgdwlhnhcx7\" to finish creating: GET https://rp.j9627648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/440484a5-098c-4647-932e-d9ebc38c3bc1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/440484a5-098c-4647-932e-d9ebc38c3bc1\",\n \"name\": \"440484a5-098c-4647-932e-d9ebc38c3bc1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T12:55:54.30559047Z\",\n \"endTime\": \"2026-09-17T13:24:59.62081715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 19m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-jlgdwlhnhcx7/cluster-candidate-5-0-h6gm8t should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00088e480\u003e: \n failed to create HCP cluster cluster-candidate-5-0-h6gm8t: failed waiting for cluster=\"cluster-candidate-5-0-h6gm8t\" in resourcegroup=\"rg-candidate-5-0-jlgdwlhnhcx7\" to finish creating: GET https://rp.j9627648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/440484a5-098c-4647-932e-d9ebc38c3bc1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/440484a5-098c-4647-932e-d9ebc38c3bc1\",\n \"name\": \"440484a5-098c-4647-932e-d9ebc38c3bc1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T12:55:54.30559047Z\",\n \"endTime\": \"2026-09-17T13:24:59.62081715Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 19m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6990,"pr_state":"closed","pr_sha":"d99f078a75aa0306d1ca53ed6e0f00bba4a7cb9a","final_merged_sha":"a0902051078fd0481b53edb539f46458fe329aca","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T11:56:43Z","started_at":"2026-09-17T11:56:43Z","completed_at":"2026-09-17T14:02:05Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:13.405956279Z","timing_metadata_checked_at":"2026-09-18T15:41:13.405956279Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:13.409683693Z","region_metadata_checked_at":"2026-09-18T15:41:13.409683693Z"},"failed_at":["alert","e2e"],"failed_test_count":12,"occurrences":[{"row_id":"0a873e7714810a2b22ea4ee531f6183a6f0cfd1500c552406ea2c964be3d3784","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"fe2081edacd08521e3b4731165e95ed1b4f17a242e9810e3d19e5b07192590fb","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-vcz7fgzqgtdm/cluster-candidate-5-0-ztqsnb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000842d80\u003e: \n failed to create HCP cluster cluster-candidate-5-0-ztqsnb: failed waiting for cluster=\"cluster-candidate-5-0-ztqsnb\" in resourcegroup=\"rg-candidate-5-0-vcz7fgzqgtdm\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8772269d-dde0-426b-b30c-d1cd81cfba32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8772269d-dde0-426b-b30c-d1cd81cfba32\",\n \"name\": \"8772269d-dde0-426b-b30c-d1cd81cfba32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:09.279842141Z\",\n \"endTime\": \"2026-09-17T13:40:12.356762691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 20m53s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-vcz7fgzqgtdm/cluster-candidate-5-0-ztqsnb should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000842d80\u003e: \n failed to create HCP cluster cluster-candidate-5-0-ztqsnb: failed waiting for cluster=\"cluster-candidate-5-0-ztqsnb\" in resourcegroup=\"rg-candidate-5-0-vcz7fgzqgtdm\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8772269d-dde0-426b-b30c-d1cd81cfba32\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8772269d-dde0-426b-b30c-d1cd81cfba32\",\n \"name\": \"8772269d-dde0-426b-b30c-d1cd81cfba32\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:09.279842141Z\",\n \"endTime\": \"2026-09-17T13:40:12.356762691Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 20m53s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0c96722926a6d012780a0620118d4e45a41e103c61fe8447e0ee52b5adf8946c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"f0d309a90401bc7c464e4f86124fff842aa1f093303fd8bfa28b66ac92405fe2","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc0560\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w9w5xmhcpq9k\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea230971-c3c1-419c-ac11-4d8ec5c1d9ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea230971-c3c1-419c-ac11-4d8ec5c1d9ef\",\n \"name\": \"ea230971-c3c1-419c-ac11-4d8ec5c1d9ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:41.071539907Z\",\n \"endTime\": \"2026-09-17T13:40:42.303659137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc0560\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-w9w5xmhcpq9k\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea230971-c3c1-419c-ac11-4d8ec5c1d9ef\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ea230971-c3c1-419c-ac11-4d8ec5c1d9ef\",\n \"name\": \"ea230971-c3c1-419c-ac11-4d8ec5c1d9ef\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:41.071539907Z\",\n \"endTime\": \"2026-09-17T13:40:42.303659137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2172743ab25534404f2fc9f5c14678d77a4de3a0540a0a6b6f15f393f301fc96","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"2cb0fa49d019c2b0a11abf2d3312cdc656ba3974fcf4c96aa2f18765ffe8dee0","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-lpqjxws5tpv6/cluster-candidate-4-21-79bc8q should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001192360\u003e: \n failed to create HCP cluster cluster-candidate-4-21-79bc8q: failed waiting for cluster=\"cluster-candidate-4-21-79bc8q\" in resourcegroup=\"rg-candidate-4-21-lpqjxws5tpv6\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e52bdd9-b621-4bdf-8c7f-88803b26d4b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e52bdd9-b621-4bdf-8c7f-88803b26d4b1\",\n \"name\": \"0e52bdd9-b621-4bdf-8c7f-88803b26d4b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:08.998702999Z\",\n \"endTime\": \"2026-09-17T13:40:12.3101134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-lpqjxws5tpv6/cluster-candidate-4-21-79bc8q should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001192360\u003e: \n failed to create HCP cluster cluster-candidate-4-21-79bc8q: failed waiting for cluster=\"cluster-candidate-4-21-79bc8q\" in resourcegroup=\"rg-candidate-4-21-lpqjxws5tpv6\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e52bdd9-b621-4bdf-8c7f-88803b26d4b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0e52bdd9-b621-4bdf-8c7f-88803b26d4b1\",\n \"name\": \"0e52bdd9-b621-4bdf-8c7f-88803b26d4b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:08.998702999Z\",\n \"endTime\": \"2026-09-17T13:40:12.3101134Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m28s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"245df161864b9e2826142e273143d6391bb363624f712c9e1e44a5c21d85db30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"d7b3a6738a3142cf28e35b425618e47c2d78a5ec741df6019ce6ae2b560d9004","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015ec480\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4llzpkxqdt5h\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n \"name\": \"98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:12:29.574806708Z\",\n \"endTime\": \"2026-09-17T13:41:32.177488403Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0015ec480\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4llzpkxqdt5h\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n \"name\": \"98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:12:29.574806708Z\",\n \"endTime\": \"2026-09-17T13:41:32.177488403Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3259b9d3e54c650c2c83393b34b5af5b86257a2d48464a74348436a6b8a281b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"969e77e201e0ea5c3383048290794e2481a2bc4e0d76aac6da2b54ff4ff6694c","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001076240\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-zw6vh2x4vdzh\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8da3fbe2-2951-4156-b4f5-0f409aecd554\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8da3fbe2-2951-4156-b4f5-0f409aecd554\",\n \"name\": \"8da3fbe2-2951-4156-b4f5-0f409aecd554\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:15:01.412052631Z\",\n \"endTime\": \"2026-09-17T13:44:02.315476713Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m49s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001076240\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-zw6vh2x4vdzh\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8da3fbe2-2951-4156-b4f5-0f409aecd554\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8da3fbe2-2951-4156-b4f5-0f409aecd554\",\n \"name\": \"8da3fbe2-2951-4156-b4f5-0f409aecd554\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:15:01.412052631Z\",\n \"endTime\": \"2026-09-17T13:44:02.315476713Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m49s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3799af71b786246975678f700482789dfa09b1713c5f07abc24a43aeff4744f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"61b54a22bdc6f8db1004cb461b9c1ea0ce8e7bb226818ce98016843218649c01","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-2t7ghvmhbmnj/cluster-candidate-4-22-2x7d82 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001158140\u003e: \n failed to create HCP cluster cluster-candidate-4-22-2x7d82: failed waiting for cluster=\"cluster-candidate-4-22-2x7d82\" in resourcegroup=\"rg-candidate-4-22-2t7ghvmhbmnj\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4da719d0-f4ed-4162-a3ff-e4a6c5af51ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4da719d0-f4ed-4162-a3ff-e4a6c5af51ba\",\n \"name\": \"4da719d0-f4ed-4162-a3ff-e4a6c5af51ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:28.390954588Z\",\n \"endTime\": \"2026-09-17T13:40:32.344644366Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 19m22s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-2t7ghvmhbmnj/cluster-candidate-4-22-2x7d82 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001158140\u003e: \n failed to create HCP cluster cluster-candidate-4-22-2x7d82: failed waiting for cluster=\"cluster-candidate-4-22-2x7d82\" in resourcegroup=\"rg-candidate-4-22-2t7ghvmhbmnj\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4da719d0-f4ed-4162-a3ff-e4a6c5af51ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4da719d0-f4ed-4162-a3ff-e4a6c5af51ba\",\n \"name\": \"4da719d0-f4ed-4162-a3ff-e4a6c5af51ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:28.390954588Z\",\n \"endTime\": \"2026-09-17T13:40:32.344644366Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 19m22s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a20ab9febbb5c282430584688c718e1fd7ed3541f1cef41f8a9e804cfef84576","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"6989180271f1d05831c6e509c6cc427effaad2b85f3bbba286006443200fcb35","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a22d00\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-h72955nqztzc\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adbc0b31-5bf0-425f-91ac-209bee804e12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adbc0b31-5bf0-425f-91ac-209bee804e12\",\n \"name\": \"adbc0b31-5bf0-425f-91ac-209bee804e12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:42.554116144Z\",\n \"endTime\": \"2026-09-17T13:40:42.348888097Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a22d00\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-h72955nqztzc\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adbc0b31-5bf0-425f-91ac-209bee804e12\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/adbc0b31-5bf0-425f-91ac-209bee804e12\",\n \"name\": \"adbc0b31-5bf0-425f-91ac-209bee804e12\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:42.554116144Z\",\n \"endTime\": \"2026-09-17T13:40:42.348888097Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m2s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ab0e51c08a287f5ef99ead0d3ab3c8583072c3b9870b21c84b58c11b39e88700","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"7cf6bbc9da68baad7e4462e2758e70a60f2061703be4a96f8198d3c3759e5683","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--r8vk8fhz9wm8/cluster-hs-pre-nrrplm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4e3a0\u003e: \n failed to create HCP cluster cluster-hs-pre-nrrplm: failed waiting for cluster=\"cluster-hs-pre-nrrplm\" in resourcegroup=\"rg-hs-pre--r8vk8fhz9wm8\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f963a38d-9e37-48b7-bd4b-6b88e16b85ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f963a38d-9e37-48b7-bd4b-6b88e16b85ab\",\n \"name\": \"f963a38d-9e37-48b7-bd4b-6b88e16b85ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:28.528887492Z\",\n \"endTime\": \"2026-09-17T13:40:32.181099978Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m54s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--r8vk8fhz9wm8/cluster-hs-pre-nrrplm should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4e3a0\u003e: \n failed to create HCP cluster cluster-hs-pre-nrrplm: failed waiting for cluster=\"cluster-hs-pre-nrrplm\" in resourcegroup=\"rg-hs-pre--r8vk8fhz9wm8\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f963a38d-9e37-48b7-bd4b-6b88e16b85ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f963a38d-9e37-48b7-bd4b-6b88e16b85ab\",\n \"name\": \"f963a38d-9e37-48b7-bd4b-6b88e16b85ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:28.528887492Z\",\n \"endTime\": \"2026-09-17T13:40:32.181099978Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m54s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b02d4902d87658ccd4d08bfa4888bb5152877e8635f36fc7a0e1a081fff47e90","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"alert","signature_id":"35bdd9c6de24d9640b64ae41e61a7affc59db651a49ce7a2fb22727cdcad6dc9","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T13:25:00Z\n Ended: 2026-09-17T13:52:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2573056-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssphrhuflst4cpf4hbnhn709qfh15fp-p4k0t8h9d2i2s1j\", pod=\"router-68544bfbb8-v8gnr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssphrhuflst4cpf4hbnhn709qfh15fp-p4k0t8h9d2i2s1j/router-68544bfbb8-v8gnr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"b36250b59226650f022c8249c3138669bb9b17a8fcdc14a631b202a8c24afa38","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"a21f245f7a5f77e335623baa9b0ddbede34c06b521c1c7c4d490e127ce4037e7","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008688c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-lpsw2q8vdb2m\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76616663-2962-4c27-a780-127cb9b30572\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76616663-2962-4c27-a780-127cb9b30572\",\n \"name\": \"76616663-2962-4c27-a780-127cb9b30572\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:07.25067562Z\",\n \"endTime\": \"2026-09-17T13:40:12.357476827Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008688c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-lpsw2q8vdb2m\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76616663-2962-4c27-a780-127cb9b30572\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76616663-2962-4c27-a780-127cb9b30572\",\n \"name\": \"76616663-2962-4c27-a780-127cb9b30572\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:11:07.25067562Z\",\n \"endTime\": \"2026-09-17T13:40:12.357476827Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ce2ca1def9e700a4eca8ebb528fd4f8476b6d41d989644af2c40011cad1a6146","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"78cd64b525be75a4d9388fde337bd307c022a2d8ce21d0ffc33bf9360fed27a3","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083a980\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-bjdtmlhkdlb5\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4d64791-d5a0-4688-8e91-8220f53405bc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4d64791-d5a0-4688-8e91-8220f53405bc\",\n \"name\": \"e4d64791-d5a0-4688-8e91-8220f53405bc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:13:32.702937524Z\",\n \"endTime\": \"2026-09-17T13:42:32.270930101Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00083a980\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-bjdtmlhkdlb5\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4d64791-d5a0-4688-8e91-8220f53405bc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e4d64791-d5a0-4688-8e91-8220f53405bc\",\n \"name\": \"e4d64791-d5a0-4688-8e91-8220f53405bc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:13:32.702937524Z\",\n \"endTime\": \"2026-09-17T13:42:32.270930101Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ff9e4a35fb3350cdbdbf03e21e64c75595d650b884700d2a7d847f3dd977641a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","failed_at":"e2e","signature_id":"cd184104708d45339d6bffb3c0990b1efb5eafdada1ae446f6ffcf492fc8f488","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aea680\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-kmkdlhnxm6qn\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/81708d5e-7bcb-4064-a523-2f2429ca57a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/81708d5e-7bcb-4064-a523-2f2429ca57a4\",\n \"name\": \"81708d5e-7bcb-4064-a523-2f2429ca57a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:12:09.723635539Z\",\n \"endTime\": \"2026-09-17T13:41:12.177995057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000aea680\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-kmkdlhnxm6qn\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/81708d5e-7bcb-4064-a523-2f2429ca57a4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/81708d5e-7bcb-4064-a523-2f2429ca57a4\",\n \"name\": \"81708d5e-7bcb-4064-a523-2f2429ca57a4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:12:09.723635539Z\",\n \"endTime\": \"2026-09-17T13:41:12.177995057Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m59s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100553657568202752","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"open","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T11:53:12Z","started_at":"2026-09-17T11:53:12Z","completed_at":"2026-09-17T14:01:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:26.309966666Z","timing_metadata_checked_at":"2026-09-18T15:41:26.309966666Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:26.312925277Z","region_metadata_checked_at":"2026-09-18T15:41:26.312925277Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"f788c401931dc9989984e3cad3a9b055f6633eea8e7636af675e9a61bcc5ce01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100553657568202752","occurred_at":"2026-09-17T11:53:12Z","failed_at":"e2e","signature_id":"e9fcf1ca8ec5a4e5b455439c42c7c5bdd9023eaf9099b980ad11a8e344c2334d","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e98600\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-85x5lk\" in resourcegroup=\"rg-candidate-4-20-vdg5pfb8x96b\" to finish deleting: GET https://rp.j8202752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3be4750b-6d04-4962-b270-180837a66720\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3be4750b-6d04-4962-b270-180837a66720\",\n \"name\": \"3be4750b-6d04-4962-b270-180837a66720\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:08:22.9987657Z\",\n \"endTime\": \"2026-09-17T13:32:25.907204967Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7n7f9kn1tn0m8nhdclc4oi11ms50 still exists (deletion dispatched at 2026-09-17T13:08:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc000e98600\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-20-85x5lk\" in resourcegroup=\"rg-candidate-4-20-vdg5pfb8x96b\" to finish deleting: GET https://rp.j8202752.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3be4750b-6d04-4962-b270-180837a66720\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3be4750b-6d04-4962-b270-180837a66720\",\n \"name\": \"3be4750b-6d04-4962-b270-180837a66720\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T13:08:22.9987657Z\",\n \"endTime\": \"2026-09-17T13:32:25.907204967Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7n7f9kn1tn0m8nhdclc4oi11ms50 still exists (deletion dispatched at 2026-09-17T13:08:24Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100537788121223168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T10:50:09Z","started_at":"2026-09-17T10:50:09Z","completed_at":"2026-09-17T12:56:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:56.49052942Z","timing_metadata_checked_at":"2026-09-18T15:40:56.49052942Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:56.494432734Z","region_metadata_checked_at":"2026-09-18T15:40:56.494432734Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100537779011194880","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T10:50:07Z","started_at":"2026-09-17T10:50:07Z","completed_at":"2026-09-17T12:49:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:57.708388701Z","timing_metadata_checked_at":"2026-09-18T15:40:57.708388701Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:57.711404612Z","region_metadata_checked_at":"2026-09-18T15:40:57.711404612Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100537768756121600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T10:50:04Z","started_at":"2026-09-17T10:50:04Z","completed_at":"2026-09-17T12:54:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:34.565553243Z","timing_metadata_checked_at":"2026-09-18T15:40:34.565553243Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:34.568748455Z","region_metadata_checked_at":"2026-09-18T15:40:34.568748455Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"19c752aa37a9bf3d17f29440850654b9dc7af64b1da27714df939cee79f286ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100537768756121600","occurred_at":"2026-09-17T10:50:04Z","failed_at":"e2e","signature_id":"0d5854936e81c77013558f96a12c534160fb9ce058180d02b89c4d9a3544856d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac2e60\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mmb8rvzxtd75\" to finish creating: GET https://rp.j6121600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b20002d-4aaf-4a1a-8670-11b592a70d29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b20002d-4aaf-4a1a-8670-11b592a70d29\",\n \"name\": \"6b20002d-4aaf-4a1a-8670-11b592a70d29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:46:22.660045003Z\",\n \"endTime\": \"2026-09-17T12:15:31.401172503Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m23s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ac2e60\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-mmb8rvzxtd75\" to finish creating: GET https://rp.j6121600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b20002d-4aaf-4a1a-8670-11b592a70d29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6b20002d-4aaf-4a1a-8670-11b592a70d29\",\n \"name\": \"6b20002d-4aaf-4a1a-8670-11b592a70d29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:46:22.660045003Z\",\n \"endTime\": \"2026-09-17T12:15:31.401172503Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m23s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100537004893671424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"3f37770fba22a55b4336e3a85cbf275bf37c8279","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-17T10:47:02Z","started_at":"2026-09-17T10:47:02Z","completed_at":"2026-09-17T12:46:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:05.434761411Z","timing_metadata_checked_at":"2026-09-18T15:41:05.434761411Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:05.437788221Z","region_metadata_checked_at":"2026-09-18T15:41:05.437788221Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100532310913847296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6985,"pr_state":"closed","pr_sha":"31fecfea9d71150ef65e93b9941a0f9323663bdc","final_merged_sha":"d7b0b57bd15ef0bbc4edf6ad998bd65072d074ae","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T10:28:23Z","started_at":"2026-09-17T10:28:23Z","completed_at":"2026-09-17T12:27:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:09.527059358Z","timing_metadata_checked_at":"2026-09-18T15:41:09.527059358Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:09.530862072Z","region_metadata_checked_at":"2026-09-18T15:41:09.530862072Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"a28095db1b5cc7800f37951dcc9f005b6f7ce17f775ccd1bc71ce775c6c44cf6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100532310913847296","occurred_at":"2026-09-17T10:28:23Z","failed_at":"e2e","signature_id":"11862ebb0b64914c35ffbbc35bccd2d2fc728333e2922d73613bd21bacfcabf1","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8b160\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-zj6grj\" in resourcegroup=\"rg-np-version-upgrade-zj6grj-pp9qwfhhwgc8\" to finish creating: GET https://rp.j3847296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85af6203-800d-4bc5-a875-4385db2526c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85af6203-800d-4bc5-a875-4385db2526c1\",\n \"name\": \"85af6203-800d-4bc5-a875-4385db2526c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:29:25.794778495Z\",\n \"endTime\": \"2026-09-17T11:48:27.334521515Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d8b160\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-zj6grj\" in resourcegroup=\"rg-np-version-upgrade-zj6grj-pp9qwfhhwgc8\" to finish creating: GET https://rp.j3847296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85af6203-800d-4bc5-a875-4385db2526c1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85af6203-800d-4bc5-a875-4385db2526c1\",\n \"name\": \"85af6203-800d-4bc5-a875-4385db2526c1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:29:25.794778495Z\",\n \"endTime\": \"2026-09-17T11:48:27.334521515Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100527138833698816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6990,"pr_state":"closed","pr_sha":"d99f078a75aa0306d1ca53ed6e0f00bba4a7cb9a","final_merged_sha":"a0902051078fd0481b53edb539f46458fe329aca","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T10:07:50Z","started_at":"2026-09-17T10:07:50Z","completed_at":"2026-09-17T10:51:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:13.238877575Z","timing_metadata_checked_at":"2026-09-18T15:41:13.238877575Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:13.2456292Z","region_metadata_checked_at":"2026-09-18T15:41:13.2456292Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"a884bc6bee50bf7e80726d9386baa51f991ea1350111c937cb02d9d99aac0888","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100527138833698816","occurred_at":"2026-09-17T10:07:50Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6972,"pr_state":"closed","pr_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","final_merged_sha":"fa71aed8b64528142a3efab3a4d0b3f6ba172c43","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T10:01:10Z","started_at":"2026-09-17T10:01:10Z","completed_at":"2026-09-17T12:15:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:02.528146544Z","timing_metadata_checked_at":"2026-09-18T15:41:02.528146544Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:02.531547756Z","region_metadata_checked_at":"2026-09-18T15:41:02.531547756Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"0048118a6eed3749ae04d4a9b4dae75727eaaec9faf5f59c85c731637cac59c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","failed_at":"alert","signature_id":"dff627a9d4028dda60dcb2cc608e3aeb26b1fb64501996026bbb8ff831d19570","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T11:10:36Z\n Ended: 2026-09-17T11:18:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5500160-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssnii6qo5ta8ipa113mfqoa8cg9ovg6-w5z4j7t8m4l3p8k\", pod=\"router-797f64c956-w5jrz\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssnii6qo5ta8ipa113mfqoa8cg9ovg6-w5z4j7t8m4l3p8k/router-797f64c956-w5jrz has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T11:11:36Z\n Ended: 2026-09-17T11:18:36Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5500160-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssnilsu0lmer0m84rarf0f3ood2jtdl-v20261001\", pod=\"router-7dbd8fdf5f-vrx9x\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssnilsu0lmer0m84rarf0f3ood2jtdl-v20261001/router-7dbd8fdf5f-vrx9x has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T11:11:36Z\n Ended: 2026-09-17T11:39:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5500160-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssnilsu0lmer0m84rarf0f3ood2jtdl-v20261001\", pod=\"router-7dbd8fdf5f-8k8kr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssnilsu0lmer0m84rarf0f3ood2jtdl-v20261001/router-7dbd8fdf5f-8k8kr has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T11:13:34Z\n Ended: 2026-09-17T11:37:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5500160-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssnigo1rbmol44repn0nblke58e1aae-y1p9b5x6a2o0p9o\", pod=\"router-cc54f77bc-vv28z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssnigo1rbmol44repn0nblke58e1aae-y1p9b5x6a2o0p9o/router-cc54f77bc-vv28z has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"554535a6123be89c92d9fb1acf090f65249851574d4976927a448cdd21f23058","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","failed_at":"e2e","signature_id":"b8057328bdd22a04aeb32a86271770e8c39d4d2cd6eac75a438bd766471d440b","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-wvq9c9fgtszp/cluster-candidate-4-22-f59zps should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a5a5a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-f59zps: failed waiting for cluster=\"cluster-candidate-4-22-f59zps\" in resourcegroup=\"rg-candidate-4-22-wvq9c9fgtszp\" to finish creating: GET https://rp.j5500160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c01af83a-a538-473f-b3be-5e376d524286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c01af83a-a538-473f-b3be-5e376d524286\",\n \"name\": \"c01af83a-a538-473f-b3be-5e376d524286\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:56:01.960012269Z\",\n \"endTime\": \"2026-09-17T11:25:07.989351808Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-wvq9c9fgtszp/cluster-candidate-4-22-f59zps should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001a5a5a0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-f59zps: failed waiting for cluster=\"cluster-candidate-4-22-f59zps\" in resourcegroup=\"rg-candidate-4-22-wvq9c9fgtszp\" to finish creating: GET https://rp.j5500160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c01af83a-a538-473f-b3be-5e376d524286\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c01af83a-a538-473f-b3be-5e376d524286\",\n \"name\": \"c01af83a-a538-473f-b3be-5e376d524286\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:56:01.960012269Z\",\n \"endTime\": \"2026-09-17T11:25:07.989351808Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bcdf02c5c1635aab6de30e9dbb53d6040a928ce2e561453260041efce08bf5a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","failed_at":"e2e","signature_id":"3596b17636b3dd9dc619bcf3bc7003efbdc9033c334b506d05c614ad30a25383","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00061ea60\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-tn4rzxgj24f9\" to finish creating: GET https://rp.j5500160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ff5a3e5-0a3c-4580-9516-ce0a34dbd352\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ff5a3e5-0a3c-4580-9516-ce0a34dbd352\",\n \"name\": \"8ff5a3e5-0a3c-4580-9516-ce0a34dbd352\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:56:23.301058289Z\",\n \"endTime\": \"2026-09-17T11:25:27.989589306Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:88]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00061ea60\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-tn4rzxgj24f9\" to finish creating: GET https://rp.j5500160.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ff5a3e5-0a3c-4580-9516-ce0a34dbd352\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8ff5a3e5-0a3c-4580-9516-ce0a34dbd352\",\n \"name\": \"8ff5a3e5-0a3c-4580-9516-ce0a34dbd352\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:56:23.301058289Z\",\n \"endTime\": \"2026-09-17T11:25:27.989589306Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5724,"pr_state":"closed","pr_sha":"5e814b754c6ac49bdda39af9047f424419d1397c","final_merged_sha":"02f55855ad2e600a742ce1b551002c7fbd872350","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T09:59:04Z","started_at":"2026-09-17T09:59:04Z","completed_at":"2026-09-17T12:12:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:03.548409241Z","timing_metadata_checked_at":"2026-09-18T15:40:03.548409241Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:03.552730957Z","region_metadata_checked_at":"2026-09-18T15:40:03.552730957Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0fee968d71711c61961dd464d9fb82d96b2eb7965f6079c63ff0b32ff25fbadb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","failed_at":"e2e","signature_id":"b9c07fb6e82776d012cae86cfc979add2b4b19f7b344021874689105ee18c987","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-prgsd6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106cb20\u003e: \n DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-prgsd6\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00106cb20\u003e: \n DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded"}},{"row_id":"7b049fa97c0a57859768f3beda9650658b03d72a64a2aca636be3285a2518ed9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","failed_at":"e2e","signature_id":"8bdd74c61a93beb235e7d38100063c0ca3d930717abd8601b9cb4dac72149e51","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009864e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-hrxjnzlgzfg2\" to finish creating: GET https://rp.j8665344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0b9ab6a-819e-409c-b5e3-ba11f6fd4008\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0b9ab6a-819e-409c-b5e3-ba11f6fd4008\",\n \"name\": \"c0b9ab6a-819e-409c-b5e3-ba11f6fd4008\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:01:00.127507794Z\",\n \"endTime\": \"2026-09-17T11:30:02.580930241Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 25m36s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009864e0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-hrxjnzlgzfg2\" to finish creating: GET https://rp.j8665344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0b9ab6a-819e-409c-b5e3-ba11f6fd4008\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c0b9ab6a-819e-409c-b5e3-ba11f6fd4008\",\n \"name\": \"c0b9ab6a-819e-409c-b5e3-ba11f6fd4008\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:01:00.127507794Z\",\n \"endTime\": \"2026-09-17T11:30:02.580930241Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 25m36s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8a2ca6407c9287ed4b9748d18b1c40aed8db94d2af7f8f46860ab69782a078a9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","failed_at":"alert","signature_id":"a438005342e6e45a592ab0e52281d7a0356c34cf7a20d84dc4af128f220b1b66","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T11:32:30Z\n Ended: 2026-09-17T11:40:30Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8665344-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8665344-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"e064b77dbdfc4e524766500fde5eb8b5c2aeabf8051f08f13332cb96108e77c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","failed_at":"e2e","signature_id":"d6f32f2cee9bee5817a4bb9c6f37367a9340014362f5ecc9c490646d7eab987e","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-grnnjnv9wr27/cluster-candidate-4-21-fqvxll should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110c280\u003e: \n failed to create HCP cluster cluster-candidate-4-21-fqvxll: failed waiting for cluster=\"cluster-candidate-4-21-fqvxll\" in resourcegroup=\"rg-candidate-4-21-grnnjnv9wr27\" to finish creating: GET https://rp.j8665344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0d3816eb-5b98-47cf-a7d0-ab5969f2ff56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0d3816eb-5b98-47cf-a7d0-ab5969f2ff56\",\n \"name\": \"0d3816eb-5b98-47cf-a7d0-ab5969f2ff56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:00:55.316971356Z\",\n \"endTime\": \"2026-09-17T11:30:02.664970428Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 25m21s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-grnnjnv9wr27/cluster-candidate-4-21-fqvxll should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110c280\u003e: \n failed to create HCP cluster cluster-candidate-4-21-fqvxll: failed waiting for cluster=\"cluster-candidate-4-21-fqvxll\" in resourcegroup=\"rg-candidate-4-21-grnnjnv9wr27\" to finish creating: GET https://rp.j8665344.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0d3816eb-5b98-47cf-a7d0-ab5969f2ff56\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0d3816eb-5b98-47cf-a7d0-ab5969f2ff56\",\n \"name\": \"0d3816eb-5b98-47cf-a7d0-ab5969f2ff56\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:00:55.316971356Z\",\n \"endTime\": \"2026-09-17T11:30:02.664970428Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 25m21s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524896286150656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6989,"pr_state":"closed","pr_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","final_merged_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T09:58:55Z","started_at":"2026-09-17T09:58:55Z","completed_at":"2026-09-17T12:09:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.551919192Z","timing_metadata_checked_at":"2026-09-18T15:41:12.551919192Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:12.560534423Z","region_metadata_checked_at":"2026-09-18T15:41:12.560534423Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"d1e5ec290d5a657829af251318f01e6bec5c91bb0c60438a05f1219f10f93ce9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524896286150656","occurred_at":"2026-09-17T09:58:55Z","failed_at":"alert","signature_id":"fd2143a50a59cd5afd961a3854a9a94e4c9bfcd849575bf08cd24c269273e2c6","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T11:07:39Z\n Ended: 2026-09-17T11:18:41Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6150656-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssnj050ab40fgp4v0g8qea2ljm1rqu0-h3o3s1e2a8m2f5s\", pod=\"router-7c74fbf7c7-6pzzf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssnj050ab40fgp4v0g8qea2ljm1rqu0-h3o3s1e2a8m2f5s/router-7c74fbf7c7-6pzzf has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524826916556800","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6986,"pr_state":"closed","pr_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","final_merged_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T09:58:39Z","started_at":"2026-09-17T09:58:39Z","completed_at":"2026-09-17T12:16:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:11.496993579Z","timing_metadata_checked_at":"2026-09-18T15:41:11.496993579Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:11.504487006Z","region_metadata_checked_at":"2026-09-18T15:41:11.504487006Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"759b883e7b51ce0fd48508e047cedb4b5f323dd1ba7c7caea4a0984137513dc9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524826916556800","occurred_at":"2026-09-17T09:58:39Z","failed_at":"alert","signature_id":"125c7452f650c84b80c4227b5aa637ed38ecce323a860ceae503938b79787746","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T11:32:03Z\n Ended: 2026-09-17T11:40:06Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6556800-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6556800-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"764ee60fa32d166f92c6c286f96007787ee189de","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T09:58:38Z","started_at":"2026-09-17T09:58:38Z","completed_at":"2026-09-17T12:15:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:23.67531777Z","timing_metadata_checked_at":"2026-09-18T15:40:23.67531777Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:23.678556682Z","region_metadata_checked_at":"2026-09-18T15:40:23.678556682Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"c76c5ad2fcfa19d5dd1a707512fa83f7368c10b2d9925dde5fd143f7f8200940","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","failed_at":"e2e","signature_id":"b662a9003e08c3ef5e2286107d4a3536bcba752aaf78844948d28b110eee221d","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--ngdwgjgglhdj/cluster-hs-pre-g8t7gf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e380\u003e: \n failed to create HCP cluster cluster-hs-pre-g8t7gf: failed waiting for cluster=\"cluster-hs-pre-g8t7gf\" in resourcegroup=\"rg-hs-pre--ngdwgjgglhdj\" to finish creating: GET https://rp.j6993280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5612c188-57de-4476-ba6c-2ed35a77ff92\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5612c188-57de-4476-ba6c-2ed35a77ff92\",\n \"name\": \"5612c188-57de-4476-ba6c-2ed35a77ff92\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:57:35.702404296Z\",\n \"endTime\": \"2026-09-17T11:26:44.378654207Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--ngdwgjgglhdj/cluster-hs-pre-g8t7gf should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e380\u003e: \n failed to create HCP cluster cluster-hs-pre-g8t7gf: failed waiting for cluster=\"cluster-hs-pre-g8t7gf\" in resourcegroup=\"rg-hs-pre--ngdwgjgglhdj\" to finish creating: GET https://rp.j6993280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5612c188-57de-4476-ba6c-2ed35a77ff92\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5612c188-57de-4476-ba6c-2ed35a77ff92\",\n \"name\": \"5612c188-57de-4476-ba6c-2ed35a77ff92\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:57:35.702404296Z\",\n \"endTime\": \"2026-09-17T11:26:44.378654207Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d905b1b7ebc97d681c2e6c328ff9ee7d6dbb21fb94894ee694b70e5f98318536","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","failed_at":"e2e","signature_id":"7cee4e5352bcdeee9af619ecd98980da8f5a54689aadb5ebc237dc1aa40f7036","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ec6e20\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-jdc8zbmqlkj4\" to finish creating: GET https://rp.j6993280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c0864bb-c64a-426d-afa3-18a243c3bd58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c0864bb-c64a-426d-afa3-18a243c3bd58\",\n \"name\": \"9c0864bb-c64a-426d-afa3-18a243c3bd58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:59:09.348421927Z\",\n \"endTime\": \"2026-09-17T11:28:14.430649094Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ec6e20\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-jdc8zbmqlkj4\" to finish creating: GET https://rp.j6993280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c0864bb-c64a-426d-afa3-18a243c3bd58\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9c0864bb-c64a-426d-afa3-18a243c3bd58\",\n \"name\": \"9c0864bb-c64a-426d-afa3-18a243c3bd58\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:59:09.348421927Z\",\n \"endTime\": \"2026-09-17T11:28:14.430649094Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m16s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"eac8f65f1cca6bdbc67ca493b4d778ec94735e7fdb07c314368114b77198f4c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","failed_at":"e2e","signature_id":"b097d1bd6e31b2848274482e1d375642d3af07f55f2c3adb043f6e076a9933e1","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c74920\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-r4x9tc6mxhkj\" to finish creating: GET https://rp.j6993280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffe4267e-2d41-4583-961d-347b03291885\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffe4267e-2d41-4583-961d-347b03291885\",\n \"name\": \"ffe4267e-2d41-4583-961d-347b03291885\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:04:59.768364805Z\",\n \"endTime\": \"2026-09-17T11:34:04.384726912Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c74920\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-r4x9tc6mxhkj\" to finish creating: GET https://rp.j6993280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffe4267e-2d41-4583-961d-347b03291885\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ffe4267e-2d41-4583-961d-347b03291885\",\n \"name\": \"ffe4267e-2d41-4583-961d-347b03291885\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:04:59.768364805Z\",\n \"endTime\": \"2026-09-17T11:34:04.384726912Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m25s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f8db43f145508f4eebed0bd65a8839eef88c965fbbf5de1063264390dfd0354a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","failed_at":"e2e","signature_id":"837607adb29247003f557d4cb6ffdfdb8aeac843b72cf365de5b9d8cdce185b2","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e540\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fqqfkcp6hbqk\" to finish creating: GET https://rp.j6993280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dae9a503-48dc-4d52-a9af-7dea389a97eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dae9a503-48dc-4d52-a9af-7dea389a97eb\",\n \"name\": \"dae9a503-48dc-4d52-a9af-7dea389a97eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:58:56.172242762Z\",\n \"endTime\": \"2026-09-17T11:28:04.350491435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m54s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f3e540\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-fqqfkcp6hbqk\" to finish creating: GET https://rp.j6993280.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dae9a503-48dc-4d52-a9af-7dea389a97eb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/dae9a503-48dc-4d52-a9af-7dea389a97eb\",\n \"name\": \"dae9a503-48dc-4d52-a9af-7dea389a97eb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:58:56.172242762Z\",\n \"endTime\": \"2026-09-17T11:28:04.350491435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m54s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fabdf285f04340d0c1170ff58ff60d5b24e72302c7801e733a44aef979e550f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","failed_at":"alert","signature_id":"39a62cd962057ec142de0e390236317a312acabc0d41e0544a266cdbecf8c0b4","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T11:11:35Z\n Ended: 2026-09-17T11:36:32Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6993280-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssnih45s28ov98565mqvfb07gsjjimd-d4n2e8g0i2f8m2q\", pod=\"router-5f5d484d45-qwv5z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssnih45s28ov98565mqvfb07gsjjimd-d4n2e8g0i2f8m2q/router-5f5d484d45-qwv5z has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523779246526464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6879,"pr_state":"closed","pr_sha":"865e554719be3a2a33ad3c12d3fddff783af5356","final_merged_sha":"865e554719be3a2a33ad3c12d3fddff783af5356","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T09:54:29Z","started_at":"2026-09-17T09:54:29Z","completed_at":"2026-09-17T12:17:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:29.501955433Z","timing_metadata_checked_at":"2026-09-18T15:40:29.501955433Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:29.506192348Z","region_metadata_checked_at":"2026-09-18T15:40:29.506192348Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"7d5f3c1afe94bfdf5751fa44ae2ac25483e05d25215dad2bbc3802b95257ff28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523779246526464","occurred_at":"2026-09-17T09:54:29Z","failed_at":"e2e","signature_id":"7fc0697aa9693154afc06e12851d0da5c2c6857a0d678a54183094477e417af7","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010092c0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-5pzngx\" in resourcegroup=\"rg-np-version-upgrade-5pzngx-z4lzmt6f7cqd\" to finish creating: GET https://rp.j6526464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c65ff35d-8a6d-42e7-be19-ac0fc196634c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c65ff35d-8a6d-42e7-be19-ac0fc196634c\",\n \"name\": \"c65ff35d-8a6d-42e7-be19-ac0fc196634c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:09:10.677337276Z\",\n \"endTime\": \"2026-09-17T11:28:11.131937431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010092c0\u003e: \n failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-5pzngx\" in resourcegroup=\"rg-np-version-upgrade-5pzngx-z4lzmt6f7cqd\" to finish creating: GET https://rp.j6526464.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c65ff35d-8a6d-42e7-be19-ac0fc196634c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c65ff35d-8a6d-42e7-be19-ac0fc196634c\",\n \"name\": \"c65ff35d-8a6d-42e7-be19-ac0fc196634c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T11:09:10.677337276Z\",\n \"endTime\": \"2026-09-17T11:28:11.131937431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523023395196928","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":5724,"pr_state":"closed","pr_sha":"5e814b754c6ac49bdda39af9047f424419d1397c","final_merged_sha":"02f55855ad2e600a742ce1b551002c7fbd872350","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T09:51:23Z","started_at":"2026-09-17T09:51:23Z","completed_at":"2026-09-17T11:48:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:26.099262506Z","timing_metadata_checked_at":"2026-09-18T15:41:26.099262506Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:26.102616318Z","region_metadata_checked_at":"2026-09-18T15:41:26.102616318Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"7ce6c879329a603f862181a626f374ef8552f28e7832c6c4c230595b0b61ae3b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523023395196928","occurred_at":"2026-09-17T09:51:23Z","failed_at":"alert","signature_id":"98cf6a34d877442691cd61bb1e77f0e720f13f9f1c1c8cdddc6ad3341afc78ff","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T11:12:18Z\n Ended: 2026-09-17T11:18:19Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5196928-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.120:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5196928/providers/microsoft.monitor/accounts/hcps-j5196928\", namespace=\"ocm-arohcpci01-2ssn9fqkkgqtkcb1te9l2nsihdvccpop-x8g8g9w0z4n5j0k\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-hxhtp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ssn9fqkkgqtkcb1te9l2nsihdvccpop-x8g8g9w0z4n5j0k/router has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"f6b9775bc693e9a464e5c74b3c11c30d6cc755cc03977b82fd9b80beef3d7cfe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523023395196928","occurred_at":"2026-09-17T09:51:23Z","failed_at":"alert","signature_id":"360514ce5b258c6295b00ba9d2f1d4fd30911299b6a3d2dd1a60f05cf773940e","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s) (1 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T10:47:18Z\n Ended: 2026-09-17T11:18:18Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j5196928-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssn9fqkkgqtkcb1te9l2nsihdvccpop-x8g8g9w0z4n5j0k\", pod=\"router-d8f9fc476-52b4m\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssn9fqkkgqtkcb1te9l2nsihdvccpop-x8g8g9w0z4n5j0k/router-d8f9fc476-52b4m has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100520057816748032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6939,"pr_state":"closed","pr_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","final_merged_sha":"21205e598d50d6c11e8b1c53dd58fe77a809d539","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T09:39:42Z","started_at":"2026-09-17T09:39:42Z","completed_at":"2026-09-17T11:27:03Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:44.683791955Z","timing_metadata_checked_at":"2026-09-18T15:40:44.683791955Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:44.687219667Z","region_metadata_checked_at":"2026-09-18T15:40:44.687219667Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1097a5b016cb2eef0e648a6cca3b6495549d081bc23575100c978e9dc79dd373","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100520057816748032","occurred_at":"2026-09-17T09:39:42Z","failed_at":"alert","signature_id":"5472bf78238dce0fe2f94f0aff922f667d82ac0c02290e6e2cd57f8d35e3a169","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T10:34:40Z\n Ended: 2026-09-17T10:45:38Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6748032-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssn19t7jsck03qgc5evps6405bso62l-j4d3g2z6s4z7u4d\", pod=\"router-588db7c789-b5465\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssn19t7jsck03qgc5evps6405bso62l-j4d3g2z6s4z7u4d/router-588db7c789-b5465 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"3ced57492366c8868ac35436073f98ed838781ae763d1587e7c1417db50fc6c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100520057816748032","occurred_at":"2026-09-17T09:39:42Z","failed_at":"e2e","signature_id":"2cc7ca1e205a8dfaf045d2763125c668a7279c4261f08b6ddee57e833c36210a","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b110a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sbkwl8vljgpn\" to finish creating: GET https://rp.j6748032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5015369-f740-49f9-9f1d-eb267795b2e8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5015369-f740-49f9-9f1d-eb267795b2e8\",\n \"name\": \"d5015369-f740-49f9-9f1d-eb267795b2e8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:19:28.770790207Z\",\n \"endTime\": \"2026-09-17T10:48:31.935939484Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m51s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b110a0\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sbkwl8vljgpn\" to finish creating: GET https://rp.j6748032.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5015369-f740-49f9-9f1d-eb267795b2e8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d5015369-f740-49f9-9f1d-eb267795b2e8\",\n \"name\": \"d5015369-f740-49f9-9f1d-eb267795b2e8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T10:19:28.770790207Z\",\n \"endTime\": \"2026-09-17T10:48:31.935939484Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m51s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100518871080374272","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6157,"pr_state":"open","pr_sha":"3ebfa4f9e207f7b4bcee6f63d6cd6a826882e5b4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-17T09:34:59Z","started_at":"2026-09-17T09:34:59Z","completed_at":"2026-09-17T11:37:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:04.519203933Z","timing_metadata_checked_at":"2026-09-18T15:40:04.519203933Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:04.523043647Z","region_metadata_checked_at":"2026-09-18T15:40:04.523043647Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100518437548724224","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6756,"pr_state":"open","pr_sha":"a65a65d0f5af3365546d2f648ce8be93984b79b0","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-09-17T09:33:12Z","started_at":"2026-09-17T09:33:12Z","completed_at":"2026-09-17T11:32:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:20.907577612Z","timing_metadata_checked_at":"2026-09-18T15:40:20.907577612Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:20.910584523Z","region_metadata_checked_at":"2026-09-18T15:40:20.910584523Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6672,"pr_state":"closed","pr_sha":"c0e251e0cd0115b032757bcdc11b1420057b5731","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T09:28:04Z","started_at":"2026-09-17T09:28:04Z","completed_at":"2026-09-17T11:04:59Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:10.784423192Z","timing_metadata_checked_at":"2026-09-18T15:40:10.784423192Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:10.787897805Z","region_metadata_checked_at":"2026-09-18T15:40:10.787897805Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"0802796836772f8739e5411491faebcc6a82627add9313b9dd6a8821edfe6269","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","failed_at":"e2e","signature_id":"dc4d8b8d1deffa568a3e69aa51d30ab552205c7e9c9f140c68e72f873ccbd501","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae6d00\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qjxvxd7q75hr\" to finish creating: GET https://rp.j4886144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e32fd27f-bb18-4481-845a-6650b11bd21d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e32fd27f-bb18-4481-845a-6650b11bd21d\",\n \"name\": \"e32fd27f-bb18-4481-845a-6650b11bd21d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:57:31.253921777Z\",\n \"endTime\": \"2026-09-17T10:26:33.068961561Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ae6d00\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qjxvxd7q75hr\" to finish creating: GET https://rp.j4886144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e32fd27f-bb18-4481-845a-6650b11bd21d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e32fd27f-bb18-4481-845a-6650b11bd21d\",\n \"name\": \"e32fd27f-bb18-4481-845a-6650b11bd21d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:57:31.253921777Z\",\n \"endTime\": \"2026-09-17T10:26:33.068961561Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2c9f9c424a1251990bba276a810928c7426259a77744b4c99a771e0e8f57d710","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","failed_at":"e2e","signature_id":"3f28cae00bcc4e17acb25bb050d65aad4ea199a510b36c178d2d712889664cab","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b183a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-54qnc8xlwfhg\" to finish creating: GET https://rp.j4886144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e6ba17-d4f1-4462-9b43-6e52575fd8bc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e6ba17-d4f1-4462-9b43-6e52575fd8bc\",\n \"name\": \"76e6ba17-d4f1-4462-9b43-6e52575fd8bc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:57:29.922050024Z\",\n \"endTime\": \"2026-09-17T10:26:33.079182372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b183a0\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-54qnc8xlwfhg\" to finish creating: GET https://rp.j4886144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e6ba17-d4f1-4462-9b43-6e52575fd8bc\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/76e6ba17-d4f1-4462-9b43-6e52575fd8bc\",\n \"name\": \"76e6ba17-d4f1-4462-9b43-6e52575fd8bc\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:57:29.922050024Z\",\n \"endTime\": \"2026-09-17T10:26:33.079182372Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m8s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8eface4cd23263fa22ff50eda2ee084bd0489f34f70b576a5fae278c02f099d6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","failed_at":"alert","signature_id":"a26f8c2c335980bba3f6e37f89e2b89d15ca7b79b6682f883c830ab2918e5b05","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 43 time(s) (37 unknown, 6 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T10:08:08Z\n Ended: 2026-09-17T10:44:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1ocgn9okmeulgkqmo7r5v6lch6q-m0m0g4x6i7u6c3d\", pod=\"kube-state-metrics-hcp-8654bffc85-42th9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1ocgn9okmeulgkqmo7r5v6lch6q-m0m0g4x6i7u6c3d/kube-state-metrics-hcp-8654bffc85-42th9 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T10:08:08Z\n Ended: 2026-09-17T10:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn0a58o3b6d465capumlr6m7i3tmj-p3v9g4r5h1c7b3t\", pod=\"kube-state-metrics-hcp-8654bffc85-48pgr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn0a58o3b6d465capumlr6m7i3tmj-p3v9g4r5h1c7b3t/kube-state-metrics-hcp-8654bffc85-48pgr has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T10:08:08Z\n Ended: 2026-09-17T10:26:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn0d4ev99mqi5thvqom46iu60pu8o-d9l5o4t0k9i2n1x\", pod=\"kube-state-metrics-hcp-8654bffc85-6bltn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn0d4ev99mqi5thvqom46iu60pu8o-d9l5o4t0k9i2n1x/kube-state-metrics-hcp-8654bffc85-6bltn has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T10:08:08Z\n Ended: 2026-09-17T10:47:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn25h7ceu6n469ednem3dgrmd8bps-w9b7d7c0w9m1v9m\", pod=\"kube-state-metrics-hcp-8654bffc85-q8cxk\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn25h7ceu6n469ednem3dgrmd8bps-w9b7d7c0w9m1v9m/kube-state-metrics-hcp-8654bffc85-q8cxk has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-17T10:08:08Z\n Ended: 2026-09-17T10:37:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1lcq0sd7l1s8c1f4dqjhp7mthsq-b0m0a3b8d8d1s6c\", pod=\"kube-state-metrics-hcp-8654bffc85-w7zgw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1lcq0sd7l1s8c1f4dqjhp7mthsq-b0m0a3b8d8d1s6c/kube-state-metrics-hcp-8654bffc85-w7zgw has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-17T10:08:08Z\n Ended: 2026-09-17T10:44:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn2ebkk93lce8296nq51hgn6drh0s-h0v7v4i7f0r3k1n\", pod=\"kube-state-metrics-hcp-8654bffc85-455s6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn2ebkk93lce8296nq51hgn6drh0s-h0v7v4i7f0r3k1n/kube-state-metrics-hcp-8654bffc85-455s6 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-17T10:08:57Z\n Ended: 2026-09-17T10:39:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn2dtiv895o9opovjsku0grrv59ti-d1u2b7b9s9t1i6v\", pod=\"kube-state-metrics-hcp-8654bffc85-jrdhx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn2dtiv895o9opovjsku0grrv59ti-d1u2b7b9s9t1i6v/kube-state-metrics-hcp-8654bffc85-jrdhx has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-17T10:08:57Z\n Ended: 2026-09-17T10:36:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1sn5ob4vstc7h5h86vq6ul0ec9l-r4u4j1g4n7o4j5e\", pod=\"kube-state-metrics-hcp-8654bffc85-gzx69\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1sn5ob4vstc7h5h86vq6ul0ec9l-r4u4j1g4n7o4j5e/kube-state-metrics-hcp-8654bffc85-gzx69 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-17T10:08:57Z\n Ended: 2026-09-17T10:16:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1l9uim9epfk6ldii7gv21mo14ju-i0d3e1u2c7q6u2i\", pod=\"kube-state-metrics-hcp-8654bffc85-kdm8q\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1l9uim9epfk6ldii7gv21mo14ju-i0d3e1u2c7q6u2i/kube-state-metrics-hcp-8654bffc85-kdm8q has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-17T10:08:57Z\n Ended: 2026-09-17T10:30:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn0a8oetivdoevi265tngl26ffi2p-h2z5a1f2o3b0l9t\", pod=\"kube-state-metrics-hcp-8654bffc85-74m8n\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn0a8oetivdoevi265tngl26ffi2p-h2z5a1f2o3b0l9t/kube-state-metrics-hcp-8654bffc85-74m8n has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-17T10:08:57Z\n Ended: 2026-09-17T10:37:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn0dh3betp1ptm9f44oadl6fegvfm-j8p8y9g1m8c4u5q\", pod=\"kube-state-metrics-hcp-8654bffc85-4t685\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn0dh3betp1ptm9f44oadl6fegvfm-j8p8y9g1m8c4u5q/kube-state-metrics-hcp-8654bffc85-4t685 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-17T10:08:57Z\n Ended: 2026-09-17T10:23:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmmvum7ql5nct035o7hm4r4o8r150u-ea-listn6n2j7\", pod=\"kube-state-metrics-hcp-8654bffc85-zl25q\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmmvum7ql5nct035o7hm4r4o8r150u-ea-listn6n2j7/kube-state-metrics-hcp-8654bffc85-zl25q has been in a non-ready state for longer than 5 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-17T10:08:57Z\n Ended: 2026-09-17T10:31:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1c6p35ld76g44ebp5k34j1cf8qt-g8b0m5e6u5p0z3y\", pod=\"kube-state-metrics-hcp-8654bffc85-p4c6z\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1c6p35ld76g44ebp5k34j1cf8qt-g8b0m5e6u5p0z3y/kube-state-metrics-hcp-8654bffc85-p4c6z has been in a non-ready state for longer than 5 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-17T10:08:57Z\n Ended: 2026-09-17T10:24:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn014in935aglqcdgufni4ccv83p7-i2p1y4q9y9l0q5m\", pod=\"kube-state-metrics-hcp-8654bffc85-jpxbn\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn014in935aglqcdgufni4ccv83p7-i2p1y4q9y9l0q5m/kube-state-metrics-hcp-8654bffc85-jpxbn has been in a non-ready state for longer than 5 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-17T10:08:57Z\n Ended: 2026-09-17T10:40:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1mrt1i79ptk4i2pr4va3ngq3o3m-j5t3c4o7e0t3l5x\", pod=\"kube-state-metrics-hcp-8654bffc85-gsnkm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1mrt1i79ptk4i2pr4va3ngq3o3m-j5t3c4o7e0t3l5x/kube-state-metrics-hcp-8654bffc85-gsnkm has been in a non-ready state for longer than 5 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:23:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1ena1dldp86nrb26jva7n63m03k-sre-hcp-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-744rg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1ena1dldp86nrb26jva7n63m03k-sre-hcp-cluster/kube-state-metrics-hcp-8654bffc85-744rg has been in a non-ready state for longer than 5 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:40:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn3du4o1fuau7gvajqpovmd57pvk7-d4y4k6z7w3p1e4v\", pod=\"kube-state-metrics-hcp-8654bffc85-9fzq9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn3du4o1fuau7gvajqpovmd57pvk7-d4y4k6z7w3p1e4v/kube-state-metrics-hcp-8654bffc85-9fzq9 has been in a non-ready state for longer than 5 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:34:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1cg383r0cnqbhkk9rq96jb02224-ephemeral-disk\", pod=\"kube-state-metrics-hcp-8654bffc85-4sc5p\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1cg383r0cnqbhkk9rq96jb02224-ephemeral-disk/kube-state-metrics-hcp-8654bffc85-4sc5p has been in a non-ready state for longer than 5 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:44:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1lks0dla8jmptpn1827rsg91g09-u7n9o6f3e8a7w8o\", pod=\"kube-state-metrics-hcp-8654bffc85-kvcc7\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1lks0dla8jmptpn1827rsg91g09-u7n9o6f3e8a7w8o/kube-state-metrics-hcp-8654bffc85-kvcc7 has been in a non-ready state for longer than 5 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:19:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmnenguljcne093trac0vrc3vmiqif-b8u4p1e5z5l6f9f\", pod=\"kube-state-metrics-hcp-8654bffc85-mbx9d\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmnenguljcne093trac0vrc3vmiqif-b8u4p1e5z5l6f9f/kube-state-metrics-hcp-8654bffc85-mbx9d has been in a non-ready state for longer than 5 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:40:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn2fun0lvh3iuc5tcu6acte0905nh-a7v6l2i4p6l1l5z\", pod=\"kube-state-metrics-hcp-8654bffc85-w28ps\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn2fun0lvh3iuc5tcu6acte0905nh-a7v6l2i4p6l1l5z/kube-state-metrics-hcp-8654bffc85-w28ps has been in a non-ready state for longer than 5 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:28:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn36hu0p06ncif3i0ji9vedp2ec8n-oidc-wi-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-v5m62\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn36hu0p06ncif3i0ji9vedp2ec8n-oidc-wi-cluster/kube-state-metrics-hcp-8654bffc85-v5m62 has been in a non-ready state for longer than 5 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:33:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn2i8juokauhkpbv9udmlep1f7kqs-ea-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-kkfz9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn2i8juokauhkpbv9udmlep1f7kqs-ea-cluster/kube-state-metrics-hcp-8654bffc85-kkfz9 has been in a non-ready state for longer than 5 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:24:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn0csmnpuis4egfkdtn7dohu0naou-basic-cluster\", pod=\"kube-state-metrics-hcp-8654bffc85-2zh2s\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn0csmnpuis4egfkdtn7dohu0naou-basic-cluster/kube-state-metrics-hcp-8654bffc85-2zh2s has been in a non-ready state for longer than 5 minutes.\nFiring 25:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:50:59Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn2eo3ft0d2qqqcc969fi4iis8kpa-d4a2v4u3r6l4c2x\", pod=\"kube-state-metrics-hcp-8654bffc85-z6qpw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn2eo3ft0d2qqqcc969fi4iis8kpa-d4a2v4u3r6l4c2x/kube-state-metrics-hcp-8654bffc85-z6qpw has been in a non-ready state for longer than 5 minutes.\nFiring 26:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:36:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn0f7r1saveq4f93saqmmcarihbga-o3n9l2e0x5h2v6i\", pod=\"router-598fb44f9c-jxfpf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn0f7r1saveq4f93saqmmcarihbga-o3n9l2e0x5h2v6i/router-598fb44f9c-jxfpf has been in a non-ready state for longer than 5 minutes.\nFiring 27:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:27:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1bc6l7vdi3pgb6sqmm108clbehj-a8b1s1y4e6n7l6s\", pod=\"kube-state-metrics-hcp-8654bffc85-njk9j\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1bc6l7vdi3pgb6sqmm108clbehj-a8b1s1y4e6n7l6s/kube-state-metrics-hcp-8654bffc85-njk9j has been in a non-ready state for longer than 5 minutes.\nFiring 28:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:28:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn1bm3iapvu18q4u88edtt7ro8tfq-h6x3s7w6l7s0b0u\", pod=\"kube-state-metrics-hcp-8654bffc85-lj5lp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn1bm3iapvu18q4u88edtt7ro8tfq-h6x3s7w6l7s0b0u/kube-state-metrics-hcp-8654bffc85-lj5lp has been in a non-ready state for longer than 5 minutes.\nFiring 29:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:26:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn054nt2b6ab14cr4f7ba73sjotgn-j2v4q1n9j1w2o3m\", pod=\"kube-state-metrics-hcp-8654bffc85-zjhn6\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn054nt2b6ab14cr4f7ba73sjotgn-j2v4q1n9j1w2o3m/kube-state-metrics-hcp-8654bffc85-zjhn6 has been in a non-ready state for longer than 5 minutes.\nFiring 30:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:18:01Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmnel48n6tj55ort1inpjfgcloo0nb-o1y1y5i1r8x1f6z\", pod=\"kube-state-metrics-hcp-8654bffc85-5tvq8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmnel48n6tj55ort1inpjfgcloo0nb-o1y1y5i1r8x1f6z/kube-state-metrics-hcp-8654bffc85-5tvq8 has been in a non-ready state for longer than 5 minutes.\nFiring 31:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:44:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn2dvkel9cdp8tmh7djfjurd4cbb9-f7d4o7x7m2n2v8n\", pod=\"kube-state-metrics-hcp-8654bffc85-vwpl8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn2dvkel9cdp8tmh7djfjurd4cbb9-f7d4o7x7m2n2v8n/kube-state-metrics-hcp-8654bffc85-vwpl8 has been in a non-ready state for longer than 5 minutes.\nFiring 32:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:33:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn07khhr5r6rjbrjh69c3akabp0o5-i9o1j9m3z5t7w4c\", pod=\"kube-state-metrics-hcp-8654bffc85-xtdn8\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn07khhr5r6rjbrjh69c3akabp0o5-i9o1j9m3z5t7w4c/kube-state-metrics-hcp-8654bffc85-xtdn8 has been in a non-ready state for longer than 5 minutes.\nFiring 33:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:40:57Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmndjv5lflbbttaeo5260nl2skjbjs-t7j8s8i3n9h3y7e\", pod=\"kube-state-metrics-hcp-8654bffc85-vxtdf\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmndjv5lflbbttaeo5260nl2skjbjs-t7j8s8i3n9h3y7e/kube-state-metrics-hcp-8654bffc85-vxtdf has been in a non-ready state for longer than 5 minutes.\nFiring 34:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:27:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn394i64jkp8u1qr0g5bup79cthmm-z7d8o5c2y4o4w8y\", pod=\"kube-state-metrics-hcp-8654bffc85-vnrfg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn394i64jkp8u1qr0g5bup79cthmm-z7d8o5c2y4o4w8y/kube-state-metrics-hcp-8654bffc85-vnrfg has been in a non-ready state for longer than 5 minutes.\nFiring 35:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:31:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn081b3qs4ov26quq8rkkd6kmvgkv-private-ingress\", pod=\"kube-state-metrics-hcp-8654bffc85-2bvxq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn081b3qs4ov26quq8rkkd6kmvgkv-private-ingress/kube-state-metrics-hcp-8654bffc85-2bvxq has been in a non-ready state for longer than 5 minutes.\nFiring 36:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:31:58Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn2p7oql665o9t78sdk9ag3gtrap1-f5u5y6g6q2m8s6o\", pod=\"kube-state-metrics-hcp-8654bffc85-f5bph\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn2p7oql665o9t78sdk9ag3gtrap1-f5u5y6g6q2m8s6o/kube-state-metrics-hcp-8654bffc85-f5bph has been in a non-ready state for longer than 5 minutes.\nFiring 37:\n State: Resolved\n Started: 2026-09-17T10:09:57Z\n Ended: 2026-09-17T10:16:00Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4886144-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmn2qsgbh3jvrunhd5vigk8r5hcn7p-s6h4n3m0q1q5r1v\", pod=\"kube-state-metrics-hcp-8654bffc85-7zchh\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmn2qsgbh3jvrunhd5vigk8r5hcn7p-s6h4n3m0q1q5r1v/kube-state-metrics-hcp-8654bffc85-7zchh has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"90800d649be50c776684b667f65725f5d50f306e6f3dc49c05e72058ca10f2c3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","failed_at":"e2e","signature_id":"4645b1f607eafec38a4f58b597cf9ee778993ba4ba3bc944eda97fc9f0f70da7","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:97]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for cluster \"\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:95 with:\nexpected kube_node_info metrics for cluster \"\" but got no results\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:97]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for cluster \"\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:95 with:\nexpected kube_node_info metrics for cluster \"\" but got no results\nExpected\n \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cea54d0df4cc4df84a4bbaf41795235750a198ce743843fbdff1373606efa1aa","failure_pattern":"kube_node_info metrics never appeared in Azure Monitor for cluster \"\"","search_query":"kube_node_info metrics never appeared in Azure Monitor for cluster \"\""}},{"row_id":"a0bd81f289e37a85d753e0ec8e394b3daa30de85b0a923100c08db7ab409a5b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","failed_at":"e2e","signature_id":"daa3fa7c154560025dfba9dba52f2111304f2d047aafee6e951b7d85e52a0fdc","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0018b8060\u003e: \n failed to get kusto cluster name from config: configuration: key kusto not found\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc0018b8060\u003e: \n failed to get kusto cluster name from config: configuration: key kusto not found\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a","failure_pattern":"failed to get kusto cluster \u003ckusto-cluster\u003e from config: configuration: key kusto not found","search_query":"failed to get kusto cluster name from config: configuration: key kusto not found"}},{"row_id":"bafc75d8a7fde1a170697c3d180d6cb2ee658309076e5d19e01cda6f5d865f0a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","failed_at":"e2e","signature_id":"1e24a52fcfa8fdc207f5da162e136847f4db22f73c90f28694d25d387bc9480d","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j4886144 in resource group hcp-underlay-ci01-j4886144\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b90a00\u003e: \n failed to get workspace services-j4886144: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4886144/providers/Microsoft.Monitor/accounts/services-j4886144\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j4886144' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j4886144 in resource group hcp-underlay-ci01-j4886144\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001b90a00\u003e: \n failed to get workspace services-j4886144: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4886144/providers/Microsoft.Monitor/accounts/services-j4886144\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceGroupNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceGroupNotFound\",\n \"message\": \"Resource group 'hcp-underlay-ci01-j4886144' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Monitor","search_query":"ERROR CODE: ResourceGroupNotFound"}},{"row_id":"e5bbe010828a48ccde7b6bf9a4b22b94a7fdd499a5ec9ff5e37cf3f6270a72b6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","failed_at":"e2e","signature_id":"2cf84524e6ccdd0a761441daa39b744a2ba3952626e9461b7d45c15b3bcfd754","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-r5fv6cfx6sx5/cluster-candidate-4-21-bcgg5p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002b82e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-bcgg5p: failed waiting for cluster=\"cluster-candidate-4-21-bcgg5p\" in resourcegroup=\"rg-candidate-4-21-r5fv6cfx6sx5\" to finish creating: GET https://rp.j4886144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7e1fc86-dec9-4c9d-bd4b-1089ac2991b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7e1fc86-dec9-4c9d-bd4b-1089ac2991b7\",\n \"name\": \"c7e1fc86-dec9-4c9d-bd4b-1089ac2991b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:57:20.337198104Z\",\n \"endTime\": \"2026-09-17T10:26:23.169599025Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m17s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-r5fv6cfx6sx5/cluster-candidate-4-21-bcgg5p should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0002b82e0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-bcgg5p: failed waiting for cluster=\"cluster-candidate-4-21-bcgg5p\" in resourcegroup=\"rg-candidate-4-21-r5fv6cfx6sx5\" to finish creating: GET https://rp.j4886144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7e1fc86-dec9-4c9d-bd4b-1089ac2991b7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c7e1fc86-dec9-4c9d-bd4b-1089ac2991b7\",\n \"name\": \"c7e1fc86-dec9-4c9d-bd4b-1089ac2991b7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:57:20.337198104Z\",\n \"endTime\": \"2026-09-17T10:26:23.169599025Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m17s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f9d0d84bc739a09a34dea3f706631bc919934222d7f52bb6429e54413c4ea410","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","failed_at":"e2e","signature_id":"acc3649bd8151c983690e39ec8f25e1c5473a348f8d0898d6d79bd8563a49a87","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d26480\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xsdmfkrcw525\" to finish creating: GET https://rp.j4886144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ef95353-782b-41cc-8d21-8f7baa89c8c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ef95353-782b-41cc-8d21-8f7baa89c8c9\",\n \"name\": \"3ef95353-782b-41cc-8d21-8f7baa89c8c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:57:31.339099602Z\",\n \"endTime\": \"2026-09-17T10:26:32.999749225Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m4s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d26480\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-xsdmfkrcw525\" to finish creating: GET https://rp.j4886144.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ef95353-782b-41cc-8d21-8f7baa89c8c9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3ef95353-782b-41cc-8d21-8f7baa89c8c9\",\n \"name\": \"3ef95353-782b-41cc-8d21-8f7baa89c8c9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:57:31.339099602Z\",\n \"endTime\": \"2026-09-17T10:26:32.999749225Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m4s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a","cea54d0df4cc4df84a4bbaf41795235750a198ce743843fbdff1373606efa1aa"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6898/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100516560857731072","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6898,"pr_state":"closed","pr_sha":"3f0521ff633af0368f049bfead3093e31565b50f","final_merged_sha":"3f0521ff633af0368f049bfead3093e31565b50f","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T09:25:48Z","started_at":"2026-09-17T09:25:48Z","completed_at":"2026-09-17T11:31:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:35.435199171Z","timing_metadata_checked_at":"2026-09-18T15:40:35.435199171Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:35.438451383Z","region_metadata_checked_at":"2026-09-18T15:40:35.438451383Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6991/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100513591089172480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6991,"pr_state":"closed","pr_sha":"4ab1e276881031f8f3dcb58ff7fdf6d2578697e6","final_merged_sha":"4ab1e276881031f8f3dcb58ff7fdf6d2578697e6","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T09:14:00Z","started_at":"2026-09-17T09:14:00Z","completed_at":"2026-09-17T11:13:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:14.716559616Z","timing_metadata_checked_at":"2026-09-18T15:41:14.716559616Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:14.720077829Z","region_metadata_checked_at":"2026-09-18T15:41:14.720077829Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100507818527297536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T08:51:04Z","started_at":"2026-09-17T08:51:04Z","completed_at":"2026-09-17T10:43:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:34.394595629Z","timing_metadata_checked_at":"2026-09-18T15:40:34.394595629Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:34.398172441Z","region_metadata_checked_at":"2026-09-18T15:40:34.398172441Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"f790b474d67f8add6ce78d4b2caa5c4c617632515654d8f6c8ddf81011ca2187","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100507818527297536","occurred_at":"2026-09-17T08:51:04Z","failed_at":"alert","signature_id":"7e105769b349ece072f3c06670207d3d4017ec8870c2a43a5330028ceb5d52e3","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T09:42:54Z\n Ended: 2026-09-17T10:05:56Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7297536-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssmahmkh6lr5u979h67h90nmbs1to8p-k8u8t2z1g0o1j4b\", pod=\"router-5b765bf64f-lhx9q\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssmahmkh6lr5u979h67h90nmbs1to8p-k8u8t2z1g0o1j4b/router-5b765bf64f-lhx9q has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T08:45:01Z","started_at":"2026-09-17T08:45:01Z","completed_at":"2026-09-17T10:34:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:54.539672104Z","timing_metadata_checked_at":"2026-09-18T15:40:54.539672104Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:54.543195516Z","region_metadata_checked_at":"2026-09-18T15:40:54.543195516Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"63011abb2945453e461caea2cf4e2ad02981869137f4c4daffe096bc0f87bf81","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","failed_at":"alert","signature_id":"962f9008502054736ab6ef9e552851e993e6894bbd6022a1a131a064d8689aee","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T09:38:40Z\n Ended: 2026-09-17T10:01:40Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9752576-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssm928m721lk4068dgpl6613ujbla6c-d5p5f1n3p4f6a6b\", pod=\"router-54c87cf596-6dx4l\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssm928m721lk4068dgpl6613ujbla6c-d5p5f1n3p4f6a6b/router-54c87cf596-6dx4l has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"69b7efef22d4f3d02749359dd2ff8700b2398d1f6b9e8bb61885fb8a1bc51c19","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","failed_at":"e2e","signature_id":"6be75bbc95c50cbac96f78528dfa07add64f7a96b8fe35fe1510e1615554ba67","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b25000\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-528t9hkgrzh7\" to finish creating: GET https://rp.j9752576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0cd7741f-18ff-46d2-82fa-6693b55efc5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0cd7741f-18ff-46d2-82fa-6693b55efc5d\",\n \"name\": \"0cd7741f-18ff-46d2-82fa-6693b55efc5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:29:27.354777917Z\",\n \"endTime\": \"2026-09-17T09:58:36.226237209Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b25000\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-528t9hkgrzh7\" to finish creating: GET https://rp.j9752576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0cd7741f-18ff-46d2-82fa-6693b55efc5d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0cd7741f-18ff-46d2-82fa-6693b55efc5d\",\n \"name\": \"0cd7741f-18ff-46d2-82fa-6693b55efc5d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:29:27.354777917Z\",\n \"endTime\": \"2026-09-17T09:58:36.226237209Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"82dfeb56b63f75844a154987cad2cf90461da1c5ee92d7606f6bee007d084f00","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","failed_at":"e2e","signature_id":"7f2b1821d50e299f9faaaf1ec8623b52220c8b011f2741480e4d05563a5a26c3","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1c220\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hsk75k5rsxz7\" to finish creating: GET https://rp.j9752576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa3193c1-5588-48f4-a832-82d0c67fad40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa3193c1-5588-48f4-a832-82d0c67fad40\",\n \"name\": \"fa3193c1-5588-48f4-a832-82d0c67fad40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:28:04.923722812Z\",\n \"endTime\": \"2026-09-17T09:57:06.067298025Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d1c220\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-hsk75k5rsxz7\" to finish creating: GET https://rp.j9752576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa3193c1-5588-48f4-a832-82d0c67fad40\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fa3193c1-5588-48f4-a832-82d0c67fad40\",\n \"name\": \"fa3193c1-5588-48f4-a832-82d0c67fad40\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:28:04.923722812Z\",\n \"endTime\": \"2026-09-17T09:57:06.067298025Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8805cc9664e603b9d5c65b373034b2c8aecd295210ff1214d35a6b251978c46c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","failed_at":"e2e","signature_id":"2ac60a748539f98f2c88eefc3d7d9d565740acd60ee833fc9852acdd1cfcf35e","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000988920\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fhpcws56rm7z\" to finish creating: GET https://rp.j9752576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88694765-6437-49ac-aac4-0c9114cbb8f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88694765-6437-49ac-aac4-0c9114cbb8f3\",\n \"name\": \"88694765-6437-49ac-aac4-0c9114cbb8f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:28:10.326777677Z\",\n \"endTime\": \"2026-09-17T09:57:16.136118228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m53s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000988920\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-fhpcws56rm7z\" to finish creating: GET https://rp.j9752576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88694765-6437-49ac-aac4-0c9114cbb8f3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/88694765-6437-49ac-aac4-0c9114cbb8f3\",\n \"name\": \"88694765-6437-49ac-aac4-0c9114cbb8f3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:28:10.326777677Z\",\n \"endTime\": \"2026-09-17T09:57:16.136118228Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m53s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506100339052544","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T08:44:14Z","started_at":"2026-09-17T08:44:14Z","completed_at":"2026-09-17T10:35:31Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:08.339825972Z","timing_metadata_checked_at":"2026-09-18T15:41:08.339825972Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:08.343130783Z","region_metadata_checked_at":"2026-09-18T15:41:08.343130783Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100504522802597888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6990,"pr_state":"closed","pr_sha":"d99f078a75aa0306d1ca53ed6e0f00bba4a7cb9a","final_merged_sha":"a0902051078fd0481b53edb539f46458fe329aca","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T08:37:58Z","started_at":"2026-09-17T08:37:58Z","completed_at":"2026-09-17T09:22:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:13.083680914Z","timing_metadata_checked_at":"2026-09-18T15:41:13.083680914Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:13.088625532Z","region_metadata_checked_at":"2026-09-18T15:41:13.088625532Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"1edb0ccb5b3d8efe72b34341ca7c6e81f5768c88e72b405eb759c0500acbcd72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100504522802597888","occurred_at":"2026-09-17T08:37:58Z","failed_at":"provision","signature_id":"bbb80a3447e86b156cad4ed50a4810d3fa341869a205ecefaab92d1964bbcc6f","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","test_suite":"step graph","failure_text":"time=2026-09-17T09:16:31.171Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-17T09:16:31.636Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:53204456-a0a3-4fd2-89f7-36eb569bbb5d cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-ci01-j2597888] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j2597888.hcpsvc.osadev.cloud] image:map[digest:sha256:dfb4a580c4ae075df76e82458246c28e24407e254922d6c888d0d2793d3c1737 registry:arohcpsvcdev.azurecr.io repository:ci-op-6i9x21qi/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6i9x21qi/pipeline:pull workloadIdentityClientId:0ee3f5a0-d80b-4385-abcc-91385fcf6f70 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:15494d5f-cf01-4a47-ae22-344126602800 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-09-17T09:16:31.636Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:31.786Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-09-17T09:16:31.977Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-09-17T09:16:31.977Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:32.031Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:37.667Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-09-17T09:16:37.667Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"40d94d1862472748004cc33c4c935e1c5142aa767d3a1a077dd2606d1dc7c7d3","failure_pattern":"failed to roll out Helm release: Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["40d94d1862472748004cc33c4c935e1c5142aa767d3a1a077dd2606d1dc7c7d3"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100500788332728320","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6672,"pr_state":"closed","pr_sha":"c0e251e0cd0115b032757bcdc11b1420057b5731","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T08:23:07Z","started_at":"2026-09-17T08:23:07Z","completed_at":"2026-09-17T08:59:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:10.652749118Z","timing_metadata_checked_at":"2026-09-18T15:40:10.652749118Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:10.655792329Z","region_metadata_checked_at":"2026-09-18T15:40:10.655792329Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"b4558550e4bb6407196bee0a3fbfb792fa8d6192494586c7bedf968138348b0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100500788332728320","occurred_at":"2026-09-17T08:23:07Z","failed_at":"provision","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-17T08:52:09.303Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T08:52:10.143Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-17T08:52:11.020Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=e5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2728320-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8\ntime=2026-09-17T08:54:11.408Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2728320-mgmt-1/providers/Microsoft.Resources/deployments/e5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8/operationStatuses/08584119717548817375\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2728320-mgmt-1/providers/Microsoft.Resources/deployments/cluster-34wn2l5qqmupi' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/17/2026 8:52:28 AM' with correlationId 'd7f46ae1-ae89-4906-81b7-ab9a15265766', and will expire at '9/24/2026 8:52:28 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"54082ad8881c5131bff996f7153979c81faa7fa74e3b195d969ebc6eb63a2c19","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/cluster-\u003cid\u003e' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["54082ad8881c5131bff996f7153979c81faa7fa74e3b195d969ebc6eb63a2c19"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6889,"pr_state":"open","pr_sha":"5d6f262d0e011ac950fc761ba37ed90fb45dba36","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T08:16:38Z","started_at":"2026-09-17T08:16:38Z","completed_at":"2026-09-17T10:39:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:32.931035965Z","timing_metadata_checked_at":"2026-09-18T15:40:32.931035965Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:32.934436877Z","region_metadata_checked_at":"2026-09-18T15:40:32.934436877Z"},"failed_at":["alert","e2e"],"failed_test_count":7,"occurrences":[{"row_id":"05a4db920806899f9845c36078a80d67b0bf2a3edc7e3c603b468547a8da14c4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","failed_at":"e2e","signature_id":"554758499da2dbd5ca2e6de15b8c5c7a3ee81c45e3edf3c5d71f98a2d63ec2c6","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015dc168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-52qn5lk4557x\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b8b5999-969b-4fd7-a0ef-03666a4c36ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b8b5999-969b-4fd7-a0ef-03666a4c36ea\",\n \"name\": \"2b8b5999-969b-4fd7-a0ef-03666a4c36ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:16:16.071243144Z\",\n \"endTime\": \"2026-09-17T09:40:24.650679622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssluvuv49qehd1o6ihpsp6u0fhsuku7 still exists (deletion dispatched at 2026-09-17T09:16:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015dc168\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-52qn5lk4557x\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b8b5999-969b-4fd7-a0ef-03666a4c36ea\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b8b5999-969b-4fd7-a0ef-03666a4c36ea\",\n \"name\": \"2b8b5999-969b-4fd7-a0ef-03666a4c36ea\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:16:16.071243144Z\",\n \"endTime\": \"2026-09-17T09:40:24.650679622Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssluvuv49qehd1o6ihpsp6u0fhsuku7 still exists (deletion dispatched at 2026-09-17T09:16:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0a6b0973df24b758259e39dd41301694c010b192c6f2d91a190ca3baf32dc320","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","failed_at":"alert","signature_id":"a85ef3a241aa9fee6ef7ff6043d1020492ba3c7fed3de7483ade938f342c7c3e","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 10 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T09:55:32Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T10:06:32Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-17T10:16:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-17T10:16:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-17T10:19:32Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.37:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv6ma6mn5k5mt36luuom09289pq58-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-v4f24\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv6ma6mn5k5mt36luuom09289pq58-cilium-cl/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-17T10:19:32Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.37:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv6ma6mn5k5mt36luuom09289pq58-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-v4f24\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv6ma6mn5k5mt36luuom09289pq58-cilium-cl/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-17T10:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-09-17T10:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-09-17T10:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Fired (not resolved)\n Started: 2026-09-17T10:24:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"5708368e2a2ab866946ee9083be0ec7eeb18b177a520eb0bb516f9400c469024","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","failed_at":"e2e","signature_id":"724c2a9602b01afbc546852f539ff27b105da0871a0374f448eae350e85e32ce","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00101a2b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fth2ns8rql9z\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n \"name\": \"e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:27:47.803109699Z\",\n \"endTime\": \"2026-09-17T09:51:54.618583456Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv6ma6mn5k5mt36luuom09289pq58 still exists (deletion dispatched at 2026-09-17T09:27:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00101a2b8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fth2ns8rql9z\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n \"name\": \"e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:27:47.803109699Z\",\n \"endTime\": \"2026-09-17T09:51:54.618583456Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv6ma6mn5k5mt36luuom09289pq58 still exists (deletion dispatched at 2026-09-17T09:27:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8809335cb0e1a7acaf5b0fc0a985f6ea70fbec233aacaa2142a4c3988b112b9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","failed_at":"e2e","signature_id":"e88abeefa8796da83e3bcaacbe5d5b7d5cc078408bb87028a285969eda618a61","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0025dc138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2wlqnnchcnfz\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06cacc7f-3109-44fc-aa1f-226b6d9287df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06cacc7f-3109-44fc-aa1f-226b6d9287df\",\n \"name\": \"06cacc7f-3109-44fc-aa1f-226b6d9287df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:13:15.932719086Z\",\n \"endTime\": \"2026-09-17T09:37:24.699046818Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv0qgf0brht8l82lblg9u4aedu59m still exists (deletion dispatched at 2026-09-17T09:13:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0025dc138\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2wlqnnchcnfz\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06cacc7f-3109-44fc-aa1f-226b6d9287df\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06cacc7f-3109-44fc-aa1f-226b6d9287df\",\n \"name\": \"06cacc7f-3109-44fc-aa1f-226b6d9287df\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:13:15.932719086Z\",\n \"endTime\": \"2026-09-17T09:37:24.699046818Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv0qgf0brht8l82lblg9u4aedu59m still exists (deletion dispatched at 2026-09-17T09:13:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b735b2c9fdea439cbee47a8793dc31ccefa99bf453e27f84a41320f91f576ce6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","failed_at":"e2e","signature_id":"56f9b496d774ec73f62dc8457705f5fe010fad5d913e37425e1aeca77033669a","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00112c090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-8bctrm\" in resourcegroup=\"rg-zstream-upgrade-4-20-2lc8kzcjbshc\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83ec0b10-2ee0-45fb-aaa3-b482a6f739da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83ec0b10-2ee0-45fb-aaa3-b482a6f739da\",\n \"name\": \"83ec0b10-2ee0-45fb-aaa3-b482a6f739da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:36:14.84381025Z\",\n \"endTime\": \"2026-09-17T10:00:14.610296792Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv1q0m5l3l4h7q2qhv5phss5tml71 still exists (deletion dispatched at 2026-09-17T09:36:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00112c090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-8bctrm\" in resourcegroup=\"rg-zstream-upgrade-4-20-2lc8kzcjbshc\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83ec0b10-2ee0-45fb-aaa3-b482a6f739da\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83ec0b10-2ee0-45fb-aaa3-b482a6f739da\",\n \"name\": \"83ec0b10-2ee0-45fb-aaa3-b482a6f739da\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:36:14.84381025Z\",\n \"endTime\": \"2026-09-17T10:00:14.610296792Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv1q0m5l3l4h7q2qhv5phss5tml71 still exists (deletion dispatched at 2026-09-17T09:36:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ee59e69cc5a183a8b84a156d95f72c14214509e7354a56020d944428ac2ffa10","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","failed_at":"e2e","signature_id":"ff7aa9ff9517f007abe64da89b86cee0dfefa327cb83d202da998d1607130a76","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015080a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mhtb5rs5742r\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52401453-231e-453f-85cc-7914f8e3b298\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52401453-231e-453f-85cc-7914f8e3b298\",\n \"name\": \"52401453-231e-453f-85cc-7914f8e3b298\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:19:54.501526397Z\",\n \"endTime\": \"2026-09-17T09:43:54.654516896Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssluvf28mbs167aoq0cpodshs5mhpdb still exists (deletion dispatched at 2026-09-17T09:19:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015080a8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-mhtb5rs5742r\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52401453-231e-453f-85cc-7914f8e3b298\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/52401453-231e-453f-85cc-7914f8e3b298\",\n \"name\": \"52401453-231e-453f-85cc-7914f8e3b298\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:19:54.501526397Z\",\n \"endTime\": \"2026-09-17T09:43:54.654516896Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssluvf28mbs167aoq0cpodshs5mhpdb still exists (deletion dispatched at 2026-09-17T09:19:54Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"fdc09f611ae2957ad1f09c9a68963ddc5fe0db5470277d5884ed280cbc05797f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","failed_at":"e2e","signature_id":"b8fb61f35748687f1492367050e150cb136907cb120fe4d9a22bd956f2ad9b80","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015da030\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2vshj8kd4ngg\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n \"name\": \"2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:42:31.310516975Z\",\n \"endTime\": \"2026-09-17T10:06:34.577233142Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc0015da030\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2vshj8kd4ngg\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n \"name\": \"2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:42:31.310516975Z\",\n \"endTime\": \"2026-09-17T10:06:34.577233142Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":7,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342","87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862"],"matched_occurrences":7,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495912190087168","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"d7699c8d3ad4c97626062f378081da839446e831","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T08:03:45Z","started_at":"2026-09-17T08:03:45Z","completed_at":"2026-09-17T09:02:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:11.786741401Z","timing_metadata_checked_at":"2026-09-18T15:40:11.786741401Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:11.789764512Z","region_metadata_checked_at":"2026-09-18T15:40:11.789764512Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"e078cce7730d85493bc6e5cdc22f2ffda4c7377ee8541da624fbc6b08055a30f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495912190087168","occurred_at":"2026-09-17T08:03:45Z","failed_at":"provision","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","test_suite":"step graph","failure_text":"time=2026-09-17T08:26:03.998Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T08:26:04.656Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-17T08:26:05.474Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=6f88425cb7a829bd0a49692c909958854fd202dee74b8108c03378dc70806d40 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0087168-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F6f88425cb7a829bd0a49692c909958854fd202dee74b8108c03378dc70806d40\ntime=2026-09-17T08:56:03.650Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"b368522ea717841a0bf803ebbe3e8dc7455b6e5e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T08:03:06Z","started_at":"2026-09-17T08:03:06Z","completed_at":"2026-09-17T10:00:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:05.889170966Z","timing_metadata_checked_at":"2026-09-18T15:40:05.889170966Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:05.892323377Z","region_metadata_checked_at":"2026-09-18T15:40:05.892323377Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2f9cd81f9138d7e39e099577e5f257d729feb12efcee999b1c54611ac5e7ee76","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","failed_at":"e2e","signature_id":"d7f2fd6c375699dace4737f150cc1d0c82379eed84d0d265ff57b17f49ba73d8","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000f1dd40\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000f1dd40\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded"}},{"row_id":"b4f4c9b3a180dcc56447e4a98559e934e8af537fbf3893157520b87e159f6cd3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","failed_at":"alert","signature_id":"d549df554d86a3fb7f1af2edc775ca43f25de1f5239534d866f56be388d7b3fb","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 9 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T08:58:51Z\n Ended: 2026-09-17T09:07:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9433728-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sslldae8h4c7a6u4p98esr6dlbc39t7-ea-listmfx27n\", pod=\"router-76c979645c-d27x9\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sslldae8h4c7a6u4p98esr6dlbc39t7-ea-listmfx27n/router-76c979645c-d27x9 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T08:58:51Z\n Ended: 2026-09-17T09:13:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9433728-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssll9ss2j4invmhfu1opgkbei7mac58-r3u7r4x4h3m5j1z\", pod=\"router-85fc9c5d89-wp4zl\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssll9ss2j4invmhfu1opgkbei7mac58-r3u7r4x4h3m5j1z/router-85fc9c5d89-wp4zl has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T08:58:51Z\n Ended: 2026-09-17T09:21:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9433728-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssll67o6fihil7nbt3p62do423jiucc-g1u1f6v4j9t0t1c\", pod=\"router-556889d545-6hhhj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssll67o6fihil7nbt3p62do423jiucc-g1u1f6v4j9t0t1c/router-556889d545-6hhhj has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T08:58:51Z\n Ended: 2026-09-17T09:17:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9433728-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sslld54jglp10uirc5ktnad1vi9hv5g-j1j7h5z3o7a6m3n\", pod=\"router-689f7964-vlhkj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sslld54jglp10uirc5ktnad1vi9hv5g-j1j7h5z3o7a6m3n/router-689f7964-vlhkj has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-17T08:58:56Z\n Ended: 2026-09-17T09:07:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9433728-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssll8ksdacasa0ge86og97h1bkfg8g6-j9e3k5m4g8q6p6v\", pod=\"router-7b7645ddf-zfnrj\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssll8ksdacasa0ge86og97h1bkfg8g6-j9e3k5m4g8q6p6v/router-7b7645ddf-zfnrj has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-17T08:58:56Z\n Ended: 2026-09-17T09:21:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9433728-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssll8ksdacasa0ge86og97h1bkfg8g6-j9e3k5m4g8q6p6v\", pod=\"router-7b7645ddf-trkmm\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssll8ksdacasa0ge86og97h1bkfg8g6-j9e3k5m4g8q6p6v/router-7b7645ddf-trkmm has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-17T08:59:53Z\n Ended: 2026-09-17T09:21:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9433728-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssllau9fs7rn1fpd75a8c36vho4o4k0-k1j9f9n9a6h0e1v\", pod=\"router-69f966859-bzh2g\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssllau9fs7rn1fpd75a8c36vho4o4k0-k1j9f9n9a6h0e1v/router-69f966859-bzh2g has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-17T09:00:50Z\n Ended: 2026-09-17T09:07:52Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9433728-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sslla15hhhslgl3hel9urgoo1djjulm-z8t2c0r8w1p2w8f\", pod=\"router-7cb67b4d4-hf9kd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sslla15hhhslgl3hel9urgoo1djjulm-z8t2c0r8w1p2w8f/router-7cb67b4d4-hf9kd has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-17T09:00:52Z\n Ended: 2026-09-17T09:21:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9433728-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sslladsa2k556c5vj4v7ho3b6sfht8r-u0r9j5j0p2b8l3d\", pod=\"router-574c565495-9v9wv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sslladsa2k556c5vj4v7ho3b6sfht8r-u0r9j5j0p2b8l3d/router-574c565495-9v9wv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c6980a8e070397f4e05dfaaeef434f8fee0d3a8aec2cd2c64b040cf4f4f2ec5b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","failed_at":"e2e","signature_id":"d145278289b57df8d05b7f53866f25f2a3e6803008a36d7c5b0f96179504086e","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.086s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001018980\u003e: \n permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apiregistration.k8s.io/apiservices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list config.openshift.io/clusterversions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list config.openshift.io/clusteroperators: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list config.openshift.io/infrastructures: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list security.openshift.io/securitycontextconstraints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list route.openshift.io/routes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list image.openshift.io/imagestreams: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.086s.\nExpected success, but got an error:\n \u003c*fmt.wrapError | 0xc001018980\u003e: \n permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list apiregistration.k8s.io/apiservices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list config.openshift.io/clusterversions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list config.openshift.io/clusteroperators: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list config.openshift.io/infrastructures: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list security.openshift.io/securitycontextconstraints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list route.openshift.io/routes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n SSAR failed for list image.openshift.io/imagestreams: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","failure_pattern":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","search_query":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews"}},{"row_id":"e14724c125937540a76902e16df565bfd646b20fce8d2579d584e28caa427976","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","failed_at":"e2e","signature_id":"706ec129f8f00b0fced3839fa11ea395190e7f746c1a75084493342ffbc32a92","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-x2pvfg\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00184eb80\u003e: \n failed to create HCP cluster np-noedge-x2pvfg: failed waiting for cluster=\"np-noedge-x2pvfg\" in resourcegroup=\"rg-np-noedge-x2pvfg-jk75vhvpdzlr\" to finish creating: GET https://rp.j9433728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/260d3aa8-bf59-42db-ba88-b8ed4aa36bb2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/260d3aa8-bf59-42db-ba88-b8ed4aa36bb2\",\n \"name\": \"260d3aa8-bf59-42db-ba88-b8ed4aa36bb2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:45:28.535156411Z\",\n \"endTime\": \"2026-09-17T09:14:36.182618821Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 22m33s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:303]: failed to create HCP cluster np-noedge-x2pvfg\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00184eb80\u003e: \n failed to create HCP cluster np-noedge-x2pvfg: failed waiting for cluster=\"np-noedge-x2pvfg\" in resourcegroup=\"rg-np-noedge-x2pvfg-jk75vhvpdzlr\" to finish creating: GET https://rp.j9433728.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/260d3aa8-bf59-42db-ba88-b8ed4aa36bb2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/260d3aa8-bf59-42db-ba88-b8ed4aa36bb2\",\n \"name\": \"260d3aa8-bf59-42db-ba88-b8ed4aa36bb2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:45:28.535156411Z\",\n \"endTime\": \"2026-09-17T09:14:36.182618821Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.20.38 is Partial (want Completed), started 22m33s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6974,"pr_state":"closed","pr_sha":"ad18c381b0fe07c50afb22f07e5b849b9d525738","final_merged_sha":"fdb1b526d51e98ec4ae2b5a2fc9fae30dbe31d6d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T07:56:30Z","started_at":"2026-09-17T07:56:30Z","completed_at":"2026-09-17T09:55:51Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:04.209253398Z","timing_metadata_checked_at":"2026-09-18T15:41:04.209253398Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:04.21266451Z","region_metadata_checked_at":"2026-09-18T15:41:04.21266451Z"},"failed_at":["alert","e2e"],"failed_test_count":9,"occurrences":[{"row_id":"0058edfb5f69f6fd8dc235b420a946a3c113b890e0132be051613337021aafba","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","failed_at":"e2e","signature_id":"833379ae7b454d610a7efbf91b7df6a563effdab3e7cab0f6ff3de3a51cd9ae1","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000888a20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sqvbm4kfvhdf\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6dc4e658-28d5-4182-808e-588d74314272\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6dc4e658-28d5-4182-808e-588d74314272\",\n \"name\": \"6dc4e658-28d5-4182-808e-588d74314272\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:54.562496471Z\",\n \"endTime\": \"2026-09-17T09:08:55.526703283Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m10s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000888a20\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-sqvbm4kfvhdf\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6dc4e658-28d5-4182-808e-588d74314272\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6dc4e658-28d5-4182-808e-588d74314272\",\n \"name\": \"6dc4e658-28d5-4182-808e-588d74314272\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:54.562496471Z\",\n \"endTime\": \"2026-09-17T09:08:55.526703283Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m10s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"14229ae99d7b72c32a4a3d5c1dd5a857a17880aff15b552b79f6d9c79696aa2e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","failed_at":"e2e","signature_id":"32d4bd52ce4e4bf4bbb0c374a620144f8370e122a59cb1cce2ba39c24a22fcac","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-ft92xmj5rlrq/cluster-candidate-4-22-txx6kl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe4c40\u003e: \n failed to create HCP cluster cluster-candidate-4-22-txx6kl: failed waiting for cluster=\"cluster-candidate-4-22-txx6kl\" in resourcegroup=\"rg-candidate-4-22-ft92xmj5rlrq\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff6f3a6b-6c03-4c6e-abdc-bca5d9188d5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff6f3a6b-6c03-4c6e-abdc-bca5d9188d5e\",\n \"name\": \"ff6f3a6b-6c03-4c6e-abdc-bca5d9188d5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:41.649732234Z\",\n \"endTime\": \"2026-09-17T09:08:45.653917596Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-ft92xmj5rlrq/cluster-candidate-4-22-txx6kl should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fe4c40\u003e: \n failed to create HCP cluster cluster-candidate-4-22-txx6kl: failed waiting for cluster=\"cluster-candidate-4-22-txx6kl\" in resourcegroup=\"rg-candidate-4-22-ft92xmj5rlrq\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff6f3a6b-6c03-4c6e-abdc-bca5d9188d5e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ff6f3a6b-6c03-4c6e-abdc-bca5d9188d5e\",\n \"name\": \"ff6f3a6b-6c03-4c6e-abdc-bca5d9188d5e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:41.649732234Z\",\n \"endTime\": \"2026-09-17T09:08:45.653917596Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 20m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3e4215445ce89f9d2042523309e29e77feefe124c180faf6f5b6e5a46fe8830a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","failed_at":"e2e","signature_id":"88399b6110d57a9572b0144a071be4bf2ca4aeadc0f450fbc20ec223beef26bc","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008c24e0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-6whdwr7p9c2x\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b1a37f4-5980-4efe-8b59-b3d9e8dae7b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b1a37f4-5980-4efe-8b59-b3d9e8dae7b1\",\n \"name\": \"8b1a37f4-5980-4efe-8b59-b3d9e8dae7b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:41.523978916Z\",\n \"endTime\": \"2026-09-17T09:08:45.671416842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:98]: failed to create HCP cluster \"v20261001\" via v20261001preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008c24e0\u003e: \n failed to create HCP cluster v20261001: failed waiting for cluster=\"v20261001\" in resourcegroup=\"v20261001preview-6whdwr7p9c2x\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b1a37f4-5980-4efe-8b59-b3d9e8dae7b1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8b1a37f4-5980-4efe-8b59-b3d9e8dae7b1\",\n \"name\": \"8b1a37f4-5980-4efe-8b59-b3d9e8dae7b1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:41.523978916Z\",\n \"endTime\": \"2026-09-17T09:08:45.671416842Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m19s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8362f59d7ddb1b9886a7b1c94119e35f08874bc70a0db7bc732b96f9672fad72","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","failed_at":"e2e","signature_id":"71eb266b069aadc565cf2dfa5c38d0b00bec544a93f9accc8e29672f14a64cd5","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009fcd80\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-rnh4rjnz5xk5\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b9bcac2-85ba-45b9-be4a-daba9fd56b63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b9bcac2-85ba-45b9-be4a-daba9fd56b63\",\n \"name\": \"2b9bcac2-85ba-45b9-be4a-daba9fd56b63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:40:13.740985106Z\",\n \"endTime\": \"2026-09-17T09:09:15.564920315Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009fcd80\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-rnh4rjnz5xk5\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b9bcac2-85ba-45b9-be4a-daba9fd56b63\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b9bcac2-85ba-45b9-be4a-daba9fd56b63\",\n \"name\": \"2b9bcac2-85ba-45b9-be4a-daba9fd56b63\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:40:13.740985106Z\",\n \"endTime\": \"2026-09-17T09:09:15.564920315Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 15m40s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"92dd1ee744bfaca61490bd083b5dc22c55552995788dc2d4e68c8bfae5700107","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","failed_at":"alert","signature_id":"795144da2ef4d9ca390a9485406bde85898d43c117d008e0d213f31d6d5042d6","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T09:11:53Z\n Ended: 2026-09-17T09:18:57Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j3937664-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j3937664-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}},{"row_id":"9cb4d74987822468a9d58547a3017f7ef5066407ef48ff2855aae96a3cca453f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","failed_at":"e2e","signature_id":"1254becb25675dbd9ede114df6e32fd10e76cd84f176a807e3d5dc72b936cfbc","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cae9a0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bq8qv9p78d46\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80adef-bbf8-4ef2-a2e6-1b7bc53c480f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80adef-bbf8-4ef2-a2e6-1b7bc53c480f\",\n \"name\": \"1a80adef-bbf8-4ef2-a2e6-1b7bc53c480f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:15.923703924Z\",\n \"endTime\": \"2026-09-17T09:08:15.68167518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cae9a0\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-bq8qv9p78d46\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80adef-bbf8-4ef2-a2e6-1b7bc53c480f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80adef-bbf8-4ef2-a2e6-1b7bc53c480f\",\n \"name\": \"1a80adef-bbf8-4ef2-a2e6-1b7bc53c480f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:15.923703924Z\",\n \"endTime\": \"2026-09-17T09:08:15.68167518Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 22m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9c5ab8d2184d57e022ceac39b7fd23ce83ec834ff4688cc4f489ec78a14f108","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","failed_at":"e2e","signature_id":"ced0fb0de9a8798a8f89849b4e849e1ad6b42de9776ac8d0c57c01c8648db002","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001064da0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2wrs4r76bf2t\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3db4a4c9-c362-40b1-b8b6-aa852265f51b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3db4a4c9-c362-40b1-b8b6-aa852265f51b\",\n \"name\": \"3db4a4c9-c362-40b1-b8b6-aa852265f51b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:41:12.504244073Z\",\n \"endTime\": \"2026-09-17T09:10:15.531158614Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001064da0\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-2wrs4r76bf2t\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3db4a4c9-c362-40b1-b8b6-aa852265f51b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3db4a4c9-c362-40b1-b8b6-aa852265f51b\",\n \"name\": \"3db4a4c9-c362-40b1-b8b6-aa852265f51b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:41:12.504244073Z\",\n \"endTime\": \"2026-09-17T09:10:15.531158614Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m32s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"cbd93cf59f1046af4eec1448ea723ac3835cb268c54d0c44b795067f08771398","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","failed_at":"e2e","signature_id":"ffd6f4632b90a496bd463d4995c3c420f976f2007427581611b1c7bcf4dd4b9b","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e6520\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-r5d5srjmrcwt\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7cc96eb-b40c-466a-8d83-0360c997832f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7cc96eb-b40c-466a-8d83-0360c997832f\",\n \"name\": \"d7cc96eb-b40c-466a-8d83-0360c997832f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:41.812447063Z\",\n \"endTime\": \"2026-09-17T09:08:45.549760606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005e6520\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-r5d5srjmrcwt\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7cc96eb-b40c-466a-8d83-0360c997832f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d7cc96eb-b40c-466a-8d83-0360c997832f\",\n \"name\": \"d7cc96eb-b40c-466a-8d83-0360c997832f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:39:41.812447063Z\",\n \"endTime\": \"2026-09-17T09:08:45.549760606Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 20m3s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"dce1b3c41e5e86370bdbf9bea0d52eb0635efd7e3e1b9dafaaae4ab893bc1d78","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","failed_at":"e2e","signature_id":"1f2d8a520d63c686030e3b1f47535fed1c6501ab4abdd19d36fb0c76229a1cb8","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f274c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-qndsd5skx2lj\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dae3ebe-027c-47e5-aeaa-d0c024830d6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dae3ebe-027c-47e5-aeaa-d0c024830d6e\",\n \"name\": \"8dae3ebe-027c-47e5-aeaa-d0c024830d6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:43:41.343182951Z\",\n \"endTime\": \"2026-09-17T09:12:45.669886954Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f274c0\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-qndsd5skx2lj\" to finish creating: GET https://rp.j3937664.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dae3ebe-027c-47e5-aeaa-d0c024830d6e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8dae3ebe-027c-47e5-aeaa-d0c024830d6e\",\n \"name\": \"8dae3ebe-027c-47e5-aeaa-d0c024830d6e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:43:41.343182951Z\",\n \"endTime\": \"2026-09-17T09:12:45.669886954Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":9,"failure_pattern_ids":["a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":9,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"d433c21668cfdc66f98262f9099c8086b4397248","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T07:56:27Z","started_at":"2026-09-17T07:56:27Z","completed_at":"2026-09-17T10:01:17Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:20.659113018Z","timing_metadata_checked_at":"2026-09-18T15:40:20.659113018Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:20.66223163Z","region_metadata_checked_at":"2026-09-18T15:40:20.66223163Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"7b519face61d347b09626f1207fa203f6d17277ea365bf6dd0b9677150e5f5a3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","failed_at":"e2e","signature_id":"1395a5c67402b5879cd22d694e49480a23894641a58ba386a295937a83eff76f","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 654.956s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n \u003c*errors.errorString | 0xc00277e080\u003e: \n run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 654.956s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n \u003c*errors.errorString | 0xc00277e080\u003e: \n run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"62df4554fa8988de83bcea5588b7f5f69715e1db3483f50b3137f97138b59e77","failure_pattern":"run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")"}},{"row_id":"888e491fce9f4dc5c6a15aa0915197c43895dd70b6c1fe16e5f2f534e48af401","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","failed_at":"e2e","signature_id":"109feb9a36dbfe6e2dc23c6d08e79142806f38590e9a903022f2d16ea982a69f","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a21c0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rpjln49gwzjt\" to finish creating: GET https://rp.j4509824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/244583bb-acb0-496e-b778-5295c25e9e3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/244583bb-acb0-496e-b778-5295c25e9e3c\",\n \"name\": \"244583bb-acb0-496e-b778-5295c25e9e3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:48:50.334998393Z\",\n \"endTime\": \"2026-09-17T09:17:56.954213085Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m35s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a21c0\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-rpjln49gwzjt\" to finish creating: GET https://rp.j4509824.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/244583bb-acb0-496e-b778-5295c25e9e3c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/244583bb-acb0-496e-b778-5295c25e9e3c\",\n \"name\": \"244583bb-acb0-496e-b778-5295c25e9e3c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:48:50.334998393Z\",\n \"endTime\": \"2026-09-17T09:17:56.954213085Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m35s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"92d681bc94f8275cd79c3e4833617b39ae61feb45f40923cf0a2c351fc7bfa5a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","failed_at":"alert","signature_id":"371eb10e8c7a8df40477f826f6a0c01232597b190d53940fdbe6a8b4c627ede1","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T08:59:33Z\n Ended: 2026-09-17T09:19:34Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4509824-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sslmj8gp6dp7mu4eoit7s23icdlitkd-g3c4u2y6d0v6k4e\", pod=\"router-7946f56747-hpwrd\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sslmj8gp6dp7mu4eoit7s23icdlitkd-g3c4u2y6d0v6k4e/router-7946f56747-hpwrd has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T08:59:33Z\n Ended: 2026-09-17T09:18:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4509824-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sslmlo65mnav23t49hihqisn8g94q9v-o7m5c7i6o3z2x0e\", pod=\"router-55648dd5c6-cjxtr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sslmlo65mnav23t49hihqisn8g94q9v-o7m5c7i6o3z2x0e/router-55648dd5c6-cjxtr has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T09:00:34Z\n Ended: 2026-09-17T09:07:35Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4509824-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sslmquivccejp0n212gvme998ra516l-ingress-cert-le\", pod=\"router-f7754d5f6-sq8fr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sslmquivccejp0n212gvme998ra516l-ingress-cert-le/router-f7754d5f6-sq8fr has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["62df4554fa8988de83bcea5588b7f5f69715e1db3483f50b3137f97138b59e77","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T07:56:26Z","started_at":"2026-09-17T07:56:26Z","completed_at":"2026-09-17T10:05:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:46.631998662Z","timing_metadata_checked_at":"2026-09-18T15:40:46.631998662Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:46.635738075Z","region_metadata_checked_at":"2026-09-18T15:40:46.635738075Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"373af8145e54436d73daa6273998a4b8f74e99b50d966fe0fe19e71c91603160","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","failed_at":"e2e","signature_id":"38588499aa79b332fce0da4da61a0607d4dd3683eb3c9cf92c9d624bfee5f925","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--f9bkp6bbzfkc/cluster-hs-pre-kdzf97 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b10c80\u003e: \n failed to create HCP cluster cluster-hs-pre-kdzf97: failed waiting for cluster=\"cluster-hs-pre-kdzf97\" in resourcegroup=\"rg-hs-pre--f9bkp6bbzfkc\" to finish creating: GET https://rp.j0135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bbe2937a-de13-4428-8e8f-fc3414b85c27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bbe2937a-de13-4428-8e8f-fc3414b85c27\",\n \"name\": \"bbe2937a-de13-4428-8e8f-fc3414b85c27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:49:19.394818225Z\",\n \"endTime\": \"2026-09-17T09:18:28.728970828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--f9bkp6bbzfkc/cluster-hs-pre-kdzf97 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b10c80\u003e: \n failed to create HCP cluster cluster-hs-pre-kdzf97: failed waiting for cluster=\"cluster-hs-pre-kdzf97\" in resourcegroup=\"rg-hs-pre--f9bkp6bbzfkc\" to finish creating: GET https://rp.j0135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bbe2937a-de13-4428-8e8f-fc3414b85c27\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bbe2937a-de13-4428-8e8f-fc3414b85c27\",\n \"name\": \"bbe2937a-de13-4428-8e8f-fc3414b85c27\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:49:19.394818225Z\",\n \"endTime\": \"2026-09-17T09:18:28.728970828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"42d2ee01d31b3761f939cfe3d9054f0ea69641994fd55f3e219185e6f2da59e1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","failed_at":"e2e","signature_id":"41623d8231704f6b3918018ef069742c2b6b4139b736f8af0a971c826a53f099","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001365160\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-899r7cpnfdlg\" to finish creating: GET https://rp.j0135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c43c53b-7ee8-4dbf-9618-9d55032ad955\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c43c53b-7ee8-4dbf-9618-9d55032ad955\",\n \"name\": \"7c43c53b-7ee8-4dbf-9618-9d55032ad955\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:01:43.266036656Z\",\n \"endTime\": \"2026-09-17T09:20:48.655798604Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001365160\u003e: \n failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-899r7cpnfdlg\" to finish creating: GET https://rp.j0135552.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c43c53b-7ee8-4dbf-9618-9d55032ad955\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7c43c53b-7ee8-4dbf-9618-9d55032ad955\",\n \"name\": \"7c43c53b-7ee8-4dbf-9618-9d55032ad955\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T09:01:43.266036656Z\",\n \"endTime\": \"2026-09-17T09:20:48.655798604Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c495db2259282d827ce06bd233545c961fce18ba2a510a7652a338b1346c8d3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","failed_at":"alert","signature_id":"f82050e70a810fdc9a71c82fe836c8741b43edf42cceee0a76e6119d660eb530","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T09:04:47Z\n Ended: 2026-09-17T09:24:47Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j0135552-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssln47hvua3f3uv7t7n5471jgo2togc-c2z4h6w5m9z5l2w\", pod=\"router-5589fbdf8-zkfwg\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssln47hvua3f3uv7t7n5471jgo2togc-c2z4h6w5m9z5l2w/router-5589fbdf8-zkfwg has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6933/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494047901650944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6933,"pr_state":"closed","pr_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","final_merged_sha":"54311c3560cde221c1d7ecff0c12a4abcd807275","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T07:56:20Z","started_at":"2026-09-17T07:56:20Z","completed_at":"2026-09-17T10:00:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:43.82392056Z","timing_metadata_checked_at":"2026-09-18T15:40:43.82392056Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:43.827056471Z","region_metadata_checked_at":"2026-09-18T15:40:43.827056471Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488996214804480","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T07:36:16Z","started_at":"2026-09-17T07:36:16Z","completed_at":"2026-09-17T08:36:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:08.219120537Z","timing_metadata_checked_at":"2026-09-18T15:41:08.219120537Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:08.223778954Z","region_metadata_checked_at":"2026-09-18T15:41:08.223778954Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8bd37d0ee944f0254ecec3801a84e7f49121b9d272f0037935324cccdc616820","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488996214804480","occurred_at":"2026-09-17T07:36:16Z","failed_at":"provision","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","test_suite":"step graph","failure_text":"time=2026-09-17T08:00:20.528Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T08:00:21.209Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-17T08:00:22.144Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=4ef3e2deb84b2c359affabaa340d88f42db65b9a42cf6a67a1d70d0a795763f6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4804480-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4ef3e2deb84b2c359affabaa340d88f42db65b9a42cf6a67a1d70d0a795763f6\ntime=2026-09-17T08:30:20.187Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6856,"pr_state":"closed","pr_sha":"3488ce75b90ddf477eaba7e960e23f8d4d5145bc","final_merged_sha":"96f38fd5a277adea0aeec323ef5685124af19024","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T07:35:28Z","started_at":"2026-09-17T07:35:28Z","completed_at":"2026-09-17T10:00:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:28.881518201Z","timing_metadata_checked_at":"2026-09-18T15:40:28.881518201Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:28.884776813Z","region_metadata_checked_at":"2026-09-18T15:40:28.884776813Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2bd3568e47bb5358cfa1cdd9accd0d3f4df45b2bc7a57e16abc0a0172a765b77","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","failed_at":"e2e","signature_id":"dcc1d91a022ed0888281ddf9849c28aaaf8b8a73e6c45ef734d0a5c17a44b021","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:141]: failed to find private KAS internal LB IP in managed resource group \"private-kas-pzzdfb6jcgvd--managed\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001100480\u003e: \n failed to list load balancers in \"private-kas-pzzdfb6jcgvd--managed\": GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-pzzdfb6jcgvd--managed/providers/Microsoft.Network/loadBalancers\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: ResourceCollectionRequestsThrottled\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceCollectionRequestsThrottled\",\n \"message\": \"Operation 'Microsoft.Network/loadBalancers/read' failed as server encountered too many requests. Please try after '1' seconds. Tracking Id is '0bca3dc6-65f8-40d2-be6e-d4168ed4b62f'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:141]: failed to find private KAS internal LB IP in managed resource group \"private-kas-pzzdfb6jcgvd--managed\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001100480\u003e: \n failed to list load balancers in \"private-kas-pzzdfb6jcgvd--managed\": GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-pzzdfb6jcgvd--managed/providers/Microsoft.Network/loadBalancers\n --------------------------------------------------------------------------------\n RESPONSE 429: 429 Too Many Requests\n ERROR CODE: ResourceCollectionRequestsThrottled\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceCollectionRequestsThrottled\",\n \"message\": \"Operation 'Microsoft.Network/loadBalancers/read' failed as server encountered too many requests. Please try after '1' seconds. Tracking Id is '0bca3dc6-65f8-40d2-be6e-d4168ed4b62f'.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"27391593c89e1b6a3153b058ef31be166dd830240fb93e3caa733a62c02edc05","failure_pattern":"ERROR CODE: ResourceCollectionRequestsThrottled; provider Microsoft.Network","search_query":"ERROR CODE: ResourceCollectionRequestsThrottled"}},{"row_id":"7406ac6b2040843971a19ad7f1827bd5ea09487a63c303a28014c76f37325df8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"8653e974664a256f513127ef093012409d9b1ca377de1635d0b83be586e6c7cb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","failed_at":"e2e","signature_id":"d26e879298c7f737b9ae16dbe160a711548fa3f84ab3c2a0048bc582a13e4076","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014c4860\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-mnqlps6hprhr\" to finish creating: GET https://rp.j8493568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f8c19d4-6c42-4be9-873b-345d7742cc94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f8c19d4-6c42-4be9-873b-345d7742cc94\",\n \"name\": \"5f8c19d4-6c42-4be9-873b-345d7742cc94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:40:56.827824237Z\",\n \"endTime\": \"2026-09-17T09:10:02.669603289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014c4860\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-mnqlps6hprhr\" to finish creating: GET https://rp.j8493568.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f8c19d4-6c42-4be9-873b-345d7742cc94\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/5f8c19d4-6c42-4be9-873b-345d7742cc94\",\n \"name\": \"5f8c19d4-6c42-4be9-873b-345d7742cc94\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T08:40:56.827824237Z\",\n \"endTime\": \"2026-09-17T09:10:02.669603289Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m41s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"abfba1abedc462b7d599b76fd3e58fdfcc1c9d15f38d54575f7fc43bac307e24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","failed_at":"alert","signature_id":"826198aaa6fd1e4924bf9a5846057c446813d101971e8e7ab1f42a73a8fd3ad8","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T08:52:53Z\n Ended: 2026-09-17T09:38:55Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j8493568-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssliq0uldorh83h2j628oub1ngtask7-e4t5k1u0q7i8c9p\", pod=\"router-85c9f57496-xx8v2\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssliq0uldorh83h2j628oub1ngtask7-e4t5k1u0q7i8c9p/router-85c9f57496-xx8v2 has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["27391593c89e1b6a3153b058ef31be166dd830240fb93e3caa733a62c02edc05","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6977/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488650243444736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6977,"pr_state":"closed","pr_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","final_merged_sha":"557c04379e033b9de0bd85904b49103f2cdef1b1","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T07:34:54Z","started_at":"2026-09-17T07:34:54Z","completed_at":"2026-09-17T09:41:21Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:06.745177629Z","timing_metadata_checked_at":"2026-09-18T15:41:06.745177629Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:06.749996047Z","region_metadata_checked_at":"2026-09-18T15:41:06.749996047Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488384702058496","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T07:33:50Z","started_at":"2026-09-17T07:33:50Z","completed_at":"2026-09-17T09:58:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:57.583386151Z","timing_metadata_checked_at":"2026-09-18T15:40:57.583386151Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:57.586345962Z","region_metadata_checked_at":"2026-09-18T15:40:57.586345962Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"093960beafeef25cbdbc599ba1c1314e67c95e078c220459fd76c147d2e4080e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488384702058496","occurred_at":"2026-09-17T07:33:50Z","failed_at":"alert","signature_id":"77026658b7ddf2ead4aff5831cdfc9e1e01b461fa183028dccacbe6855fbe9a3","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T09:11:27Z\n Ended: 2026-09-17T09:17:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2058496-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2058496-svc).","failure_pattern_match":{"status":"clustered","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100487527256297472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T07:30:18Z","started_at":"2026-09-17T07:30:18Z","completed_at":"2026-09-17T09:51:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:25.966994229Z","timing_metadata_checked_at":"2026-09-18T15:41:25.966994229Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:25.971565545Z","region_metadata_checked_at":"2026-09-18T15:41:25.971565545Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"294bdadf501071fd5f7d2cc5561f768231e8fdfb092488f8e972fcb7ca23860f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100487527256297472","occurred_at":"2026-09-17T07:30:18Z","failed_at":"e2e","signature_id":"e98964207b5fd40111448e87aa9fa8936a8a788c4325943a9b61b7b5870f98ff","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0017a97a0\u003e: \n failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0017a97a0\u003e: \n failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100485495183118336","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6989,"pr_state":"closed","pr_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","final_merged_sha":"018e20eb4a3b8b1ff5902fdc0206f8afd7a48ada","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T07:22:21Z","started_at":"2026-09-17T07:22:21Z","completed_at":"2026-09-17T09:33:48Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:12.391915614Z","timing_metadata_checked_at":"2026-09-18T15:41:12.391915614Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:12.398139036Z","region_metadata_checked_at":"2026-09-18T15:41:12.398139036Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"53162b167b5b7dc3cef1cd3a607ef23dcc800aad4bfeec8772d06627ca71e73a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100485495183118336","occurred_at":"2026-09-17T07:22:21Z","failed_at":"alert","signature_id":"f62c0a71de4ea91a4231cade3331db31d540a634ddef3ee99f2bb1c6a99266f1","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T08:54:27Z\n Ended: 2026-09-17T09:00:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8t9zj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8t9zj (namespace arobit) on cluster ci01-j3118336-mgmt-1 has averaged 131.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T08:54:27Z\n Ended: 2026-09-17T09:10:26Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-g6d4v\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-g6d4v (namespace arobit) on cluster ci01-j3118336-mgmt-2 has averaged 183.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T08:54:27Z\n Ended: 2026-09-17T09:04:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b547f7d8b-bz92h\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7b547f7d8b-bz92h (namespace mgmt-agent) on cluster ci01-j3118336-mgmt-1 has averaged 183.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T08:54:27Z\n Ended: 2026-09-17T09:04:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c78bc956-rx5ct\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-86c78bc956-rx5ct (namespace clusters-service) on cluster ci01-j3118336-svc has averaged 829.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T08:54:27Z\n Ended: 2026-09-17T09:08:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vnkdq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-vnkdq (namespace arobit) on cluster ci01-j3118336-mgmt-1 has averaged 166.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T08:54:27Z\n Ended: 2026-09-17T09:09:27Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b547f7d8b-pnvkw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7b547f7d8b-pnvkw (namespace mgmt-agent) on cluster ci01-j3118336-mgmt-2 has averaged 166.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T08:59:26Z\n Ended: 2026-09-17T09:17:25Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c78bc956-ljk5z\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-86c78bc956-ljk5z (namespace clusters-service) on cluster ci01-j3118336-svc has averaged 226.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6791,"pr_state":"closed","pr_sha":"764ee60fa32d166f92c6c286f96007787ee189de","final_merged_sha":"00c5d478babb26a92f08e45ee9f4c322bafd7507","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T06:56:58Z","started_at":"2026-09-17T06:56:58Z","completed_at":"2026-09-17T09:02:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:23.487961796Z","timing_metadata_checked_at":"2026-09-18T15:40:23.487961796Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:23.49172301Z","region_metadata_checked_at":"2026-09-18T15:40:23.49172301Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"30c658d7e530b40dc14a6883a3cb9ad325e274eba1eeeb085d46076d7ebf2974","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","failed_at":"alert","signature_id":"aac5573ba90ecce2e869b424d24d0b2a557bdaf7fb68a59bdb1ca60dcfc91361","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T08:17:39Z\n Ended: 2026-09-17T08:27:38Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-49zq5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-49zq5 (namespace arobit) on cluster ci01-j3085056-svc has averaged 168.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T08:17:39Z\n Ended: 2026-09-17T08:27:38Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-d4b6fd9f9-p4w2z\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-d4b6fd9f9-p4w2z (namespace aro-hcp) on cluster ci01-j3085056-svc has averaged 377.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T08:17:39Z\n Ended: 2026-09-17T08:27:38Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-78587f7f-wtrr7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-78587f7f-wtrr7 (namespace clusters-service) on cluster ci01-j3085056-svc has averaged 226.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"4b05fbd285c45125ce89d9611932e927d5323b712620ac469cd08d3377a543f2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","failed_at":"e2e","signature_id":"6b862675f286ba82e4aaaeabbdde3ab8b1accc783182f06adb944ac909b9f88b","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--4qtgcwsbzxjs/cluster-hs-pre-sxxfcz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00170c380\u003e: \n failed to create HCP cluster cluster-hs-pre-sxxfcz: failed waiting for cluster=\"cluster-hs-pre-sxxfcz\" in resourcegroup=\"rg-hs-pre--4qtgcwsbzxjs\" to finish creating: GET https://rp.j3085056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/89b60f6f-1529-4076-81d5-cf78bd7c7398\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/89b60f6f-1529-4076-81d5-cf78bd7c7398\",\n \"name\": \"89b60f6f-1529-4076-81d5-cf78bd7c7398\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T07:42:50.029111515Z\",\n \"endTime\": \"2026-09-17T08:11:51.010935417Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--4qtgcwsbzxjs/cluster-hs-pre-sxxfcz should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00170c380\u003e: \n failed to create HCP cluster cluster-hs-pre-sxxfcz: failed waiting for cluster=\"cluster-hs-pre-sxxfcz\" in resourcegroup=\"rg-hs-pre--4qtgcwsbzxjs\" to finish creating: GET https://rp.j3085056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/89b60f6f-1529-4076-81d5-cf78bd7c7398\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/89b60f6f-1529-4076-81d5-cf78bd7c7398\",\n \"name\": \"89b60f6f-1529-4076-81d5-cf78bd7c7398\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T07:42:50.029111515Z\",\n \"endTime\": \"2026-09-17T08:11:51.010935417Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a9adb98960604dfbca4964ce3cd521a6a332e0df59ed70cada596e2cc0be2afb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","failed_at":"e2e","signature_id":"008a91fd3e3e5f5195f74645a3769624ed2be2ab43fcea742cd06f90df27599f","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c95880\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-542pg25pz2bb\" to finish creating: GET https://rp.j3085056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61098ffc-0e1f-4ea9-b0ff-d3f03476730b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61098ffc-0e1f-4ea9-b0ff-d3f03476730b\",\n \"name\": \"61098ffc-0e1f-4ea9-b0ff-d3f03476730b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T07:44:22.269785648Z\",\n \"endTime\": \"2026-09-17T08:13:31.026654866Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m1s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c95880\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-542pg25pz2bb\" to finish creating: GET https://rp.j3085056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61098ffc-0e1f-4ea9-b0ff-d3f03476730b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/61098ffc-0e1f-4ea9-b0ff-d3f03476730b\",\n \"name\": \"61098ffc-0e1f-4ea9-b0ff-d3f03476730b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T07:44:22.269785648Z\",\n \"endTime\": \"2026-09-17T08:13:31.026654866Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m1s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6988,"pr_state":"closed","pr_sha":"53b8fd9bee7c04069b9537ac28ccedffde355125","final_merged_sha":"53b8fd9bee7c04069b9537ac28ccedffde355125","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T06:55:45Z","started_at":"2026-09-17T06:55:45Z","completed_at":"2026-09-17T08:55:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:11.916417495Z","timing_metadata_checked_at":"2026-09-18T15:41:11.916417495Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:11.919796007Z","region_metadata_checked_at":"2026-09-18T15:41:11.919796007Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"2e10d6da4f7fd92b81f9a8b280c91a1ed48cd934f17bbc2f91c607d525f0caab","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","failed_at":"alert","signature_id":"78b86d170894f1306383271eb64e5c7231bc9db1d4dade7d27fb2bd571aeb710","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T07:58:12Z\n Ended: 2026-09-17T08:12:12Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9085312-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085312/providers/microsoft.monitor/accounts/hcps-j9085312\", namespace=\"ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-2qwp8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T07:59:15Z\n Ended: 2026-09-17T08:27:12Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9085312-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085312/providers/microsoft.monitor/accounts/hcps-j9085312\", namespace=\"ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-2qwp8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"48a7159ca0ee7c3321c33329ddfef408f6a3fc2c0d1aa00faa9436fe49f4903f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","failed_at":"alert","signature_id":"a995d8a36ccb5cf52cd9d35f8ae8324aa7b3c5eb47ebb28e229463bfb8fb04c6","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T08:13:10Z\n Ended: 2026-09-17T08:20:13Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9085312-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085312/providers/microsoft.monitor/accounts/hcps-j9085312\", namespace=\"ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-2qwp8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"bd5dccf1d01ef2f17eaa56583c116d04d6a159540e186c0a786a72ffef334ab2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","failed_at":"alert","signature_id":"62ae640dcc38eb25ad05e49560b842c80c55b99017dce7da9cea433518d78675","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T08:00:10Z\n Ended: 2026-09-17T08:08:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9085312-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085312/providers/microsoft.monitor/accounts/hcps-j9085312\", namespace=\"ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"61e3547c-41a9-4b9c-82c4-b22d00b533e0\"\n Description: Pod ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n State: Resolved\n Started: 2026-09-17T08:00:11Z\n Ended: 2026-09-17T08:24:10Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9085312-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085312/providers/microsoft.monitor/accounts/hcps-j9085312\", namespace=\"ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"61e3547c-41a9-4b9c-82c4-b22d00b533e0\"\n Description: Pod ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"f9fdcac7ad4a0f1c9f6b02ba6220887581c4fcf73bfcd81f902130167cc8d5b1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","failed_at":"e2e","signature_id":"ebdc66289f4002396961893fc85f6cbf2d72c402de90efcaab065108518df75a","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4c440\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xfhpjp2z56q6\" to finish creating: GET https://rp.j9085312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\",\n \"name\": \"0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T07:37:20.85368782Z\",\n \"endTime\": \"2026-09-17T08:06:23.778430257Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f4c440\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xfhpjp2z56q6\" to finish creating: GET https://rp.j9085312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\",\n \"name\": \"0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T07:37:20.85368782Z\",\n \"endTime\": \"2026-09-17T08:06:23.778430257Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100474592526077952","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T06:39:02Z","started_at":"2026-09-17T06:39:02Z","completed_at":"2026-09-17T08:29:24Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:25.819698898Z","timing_metadata_checked_at":"2026-09-18T15:41:25.819698898Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:25.822708909Z","region_metadata_checked_at":"2026-09-18T15:41:25.822708909Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"2c6ab70e7ac4f273b75dc7a796c3284d2b1251efd0b7b37f0de318e94ade8941","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100474592526077952","occurred_at":"2026-09-17T06:39:02Z","failed_at":"alert","signature_id":"aa91dd60d940dac3239718e70007e00a82448bdb09061df222f973e65da4b8f6","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T07:47:54Z\n Ended: 2026-09-17T07:53:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-648575ffd8-pgfs2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-648575ffd8-pgfs2 (namespace clusters-service) on cluster ci01-j6077952-svc has averaged 276.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T07:52:53Z\n Ended: 2026-09-17T08:00:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-x8286\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-x8286 (namespace arobit) on cluster ci01-j6077952-mgmt-1 has averaged 128.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T07:54:53Z\n Ended: 2026-09-17T08:01:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-p74sm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-p74sm (namespace arobit) on cluster ci01-j6077952-mgmt-1 has averaged 122.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-17T08:17:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-648575ffd8-526ht\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-648575ffd8-526ht (namespace clusters-service) on cluster ci01-j6077952-svc has averaged 185.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100472165051666432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6364,"pr_state":"open","pr_sha":"b368522ea717841a0bf803ebbe3e8dc7455b6e5e","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T06:29:23Z","started_at":"2026-09-17T06:29:23Z","completed_at":"2026-09-17T07:34:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:05.756171587Z","timing_metadata_checked_at":"2026-09-18T15:40:05.756171587Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:05.760443302Z","region_metadata_checked_at":"2026-09-18T15:40:05.760443302Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f4e7b2e2ef62e974fa5f9c8cf7671e8c836d346c116db69516ec353601fc546e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100472165051666432","occurred_at":"2026-09-17T06:29:23Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470565717413888","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6880,"pr_state":"closed","pr_sha":"2e36726e4a97495428698b756402558a81b5586a","final_merged_sha":"a20dd280dd6b7499cf853051219487314ef1e460","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T06:23:02Z","started_at":"2026-09-17T06:23:02Z","completed_at":"2026-09-17T08:30:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:30.99154819Z","timing_metadata_checked_at":"2026-09-18T15:40:30.99154819Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:30.996174406Z","region_metadata_checked_at":"2026-09-18T15:40:30.996174406Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4851b32dfc33459e0b06a5e971efc623e03a791e05a516827cd2dd5f09dd132e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470565717413888","occurred_at":"2026-09-17T06:23:02Z","failed_at":"alert","signature_id":"036570c8237e20bff362fa84f18b2757ecf6b7478b115ced0b1b6d96b52ae4ba","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T07:53:16Z\n Ended: 2026-09-17T08:10:12Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7413888-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-st8ft\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-st8ft (namespace arobit) on cluster ci01-j7413888-mgmt-1 has averaged 193.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T07:53:16Z\n Ended: 2026-09-17T08:06:16Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7413888-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-585dc64dd6-ltc9d\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-585dc64dd6-ltc9d (namespace clusters-service) on cluster ci01-j7413888-svc has averaged 242.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T07:53:16Z\n Ended: 2026-09-17T08:10:13Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7413888-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6bd4f4ffc9-zpn8q\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-6bd4f4ffc9-zpn8q (namespace mgmt-agent) on cluster ci01-j7413888-mgmt-1 has averaged 185.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T07:53:17Z\n Ended: 2026-09-17T08:02:14Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7413888-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-dqzb5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-dqzb5 (namespace arobit) on cluster ci01-j7413888-mgmt-1 has averaged 174.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470315283910656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T06:22:02Z","started_at":"2026-09-17T06:22:02Z","completed_at":"2026-09-17T08:04:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:34.190145493Z","timing_metadata_checked_at":"2026-09-18T15:40:34.190145493Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:34.195815614Z","region_metadata_checked_at":"2026-09-18T15:40:34.195815614Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"e8a8d6cd7886358505ec4a75da8fa1561e4ef595ec09a64848e740b82c4dbc40","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470315283910656","occurred_at":"2026-09-17T06:22:02Z","failed_at":"alert","signature_id":"34004eeb935c0bb4ce4ba308740f33575b771c563e1c3c5cbd660527dc370ef4","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T07:21:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-65bc959db7-ncnk8\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-65bc959db7-ncnk8 (namespace clusters-service) on cluster ci01-j3910656-svc has averaged 841.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T07:21:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-6f9547c9d5-nwkbn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-6f9547c9d5-nwkbn (namespace aro-hcp) on cluster ci01-j3910656-svc has averaged 441.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T07:21:45Z\n Ended: 2026-09-17T07:41:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7855548969-mxztp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7855548969-mxztp (namespace mgmt-agent) on cluster ci01-j3910656-mgmt-1 has averaged 162.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T07:21:45Z\n Ended: 2026-09-17T07:42:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-82s5f\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-82s5f (namespace arobit) on cluster ci01-j3910656-mgmt-1 has averaged 208.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T07:21:45Z\n Ended: 2026-09-17T07:51:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-n6dpr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-n6dpr (namespace arobit) on cluster ci01-j3910656-svc has averaged 220.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T07:22:45Z\n Ended: 2026-09-17T07:51:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-65bc959db7-fwclf\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-65bc959db7-fwclf (namespace clusters-service) on cluster ci01-j3910656-svc has averaged 249.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T07:22:45Z\n Ended: 2026-09-17T07:51:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-65bc959db7-mlkz7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-65bc959db7-mlkz7 (namespace clusters-service) on cluster ci01-j3910656-svc has averaged 264.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T07:22:45Z\n Ended: 2026-09-17T07:39:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-jb25r\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-jb25r (namespace arobit) on cluster ci01-j3910656-mgmt-2 has averaged 153.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Resolved\n Started: 2026-09-17T07:23:45Z\n Ended: 2026-09-17T07:41:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7855548969-fmbtf\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7855548969-fmbtf (namespace mgmt-agent) on cluster ci01-j3910656-mgmt-2 has averaged 165.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T07:24:47Z\n Ended: 2026-09-17T07:41:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-tpp5g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-tpp5g (namespace arobit) on cluster ci01-j3910656-mgmt-1 has averaged 144.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-17T07:27:46Z\n Ended: 2026-09-17T07:38:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-lzpht\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-lzpht (namespace arobit) on cluster ci01-j3910656-mgmt-2 has averaged 132.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n State: Resolved\n Started: 2026-09-17T07:28:45Z\n Ended: 2026-09-17T07:38:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3910656-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8thwc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8thwc (namespace arobit) on cluster ci01-j3910656-mgmt-1 has averaged 128% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100469063225446400","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6986,"pr_state":"closed","pr_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","final_merged_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T06:17:04Z","started_at":"2026-09-17T06:17:04Z","completed_at":"2026-09-17T08:05:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:10.90247393Z","timing_metadata_checked_at":"2026-09-18T15:41:10.90247393Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:10.910443559Z","region_metadata_checked_at":"2026-09-18T15:41:10.910443559Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"6a7d7e409c54bf39f3297d79e7b1bfb69cb5c666204be18a19a02d9c8a3709b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100469063225446400","occurred_at":"2026-09-17T06:17:04Z","failed_at":"alert","signature_id":"07f59a2e15e5179fe3fd25749784f754d222157d8b27384b9e1cd64842ae8768","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:43:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7f4679bf87-cxtkn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7f4679bf87-cxtkn (namespace mgmt-agent) on cluster ci01-j5446400-mgmt-1 has averaged 215.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T07:24:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-67d4567fb5-9xjlt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-67d4567fb5-9xjlt (namespace clusters-service) on cluster ci01-j5446400-svc has averaged 826.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:51:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-67d4567fb5-pd5z9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-67d4567fb5-pd5z9 (namespace clusters-service) on cluster ci01-j5446400-svc has averaged 286.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:38:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-dcdd2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-dcdd2 (namespace arobit) on cluster ci01-j5446400-mgmt-2 has averaged 149.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:42:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-swmtl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-swmtl (namespace arobit) on cluster ci01-j5446400-mgmt-2 has averaged 154.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:37:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8m78k\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8m78k (namespace arobit) on cluster ci01-j5446400-mgmt-2 has averaged 136.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:43:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7f4679bf87-jrw4v\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7f4679bf87-jrw4v (namespace mgmt-agent) on cluster ci01-j5446400-mgmt-2 has averaged 191.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Fired (not resolved)\n Started: 2026-09-17T07:24:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-64877d49d4-64ffd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-64877d49d4-64ffd (namespace aro-hcp) on cluster ci01-j5446400-svc has averaged 458.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:51:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-67d4567fb5-qj7mp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-67d4567fb5-qj7mp (namespace clusters-service) on cluster ci01-j5446400-svc has averaged 256.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:30:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-xfwt6\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-xfwt6 (namespace arobit) on cluster ci01-j5446400-mgmt-1 has averaged 183.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:36:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-t9csm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-t9csm (namespace arobit) on cluster ci01-j5446400-svc has averaged 130.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:46:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-9cw8p\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-9cw8p (namespace arobit) on cluster ci01-j5446400-svc has averaged 196.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 13:\n State: Resolved\n Started: 2026-09-17T07:24:05Z\n Ended: 2026-09-17T07:40:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5446400-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-j4w7g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-j4w7g (namespace arobit) on cluster ci01-j5446400-mgmt-1 has averaged 182.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"7d702444a35f92aa3e653a67cba646c3b78ba6b991907a7e81b63be06c93e2b0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100469063225446400","occurred_at":"2026-09-17T06:17:04Z","failed_at":"e2e","signature_id":"41037677a8ddb37389a2d414387cf88a833a474c6bb9aecf6b5fe7288cd9a6b8","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:598]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001f1640\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-tphl5x\" in resourcegroup=\"rg-np-downgrade-tphl5x-crdlsng97qjz\" to finish creating: GET https://rp.j5446400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/277c2008-b4c1-4e9e-b6d0-fc1bfebe17ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/277c2008-b4c1-4e9e-b6d0-fc1bfebe17ba\",\n \"name\": \"277c2008-b4c1-4e9e-b6d0-fc1bfebe17ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T07:01:20.069431152Z\",\n \"endTime\": \"2026-09-17T07:20:25.508659948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:598]: failed to create nodepool npdg-4-21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001f1640\u003e: \n failed to create NodePool npdg-4-21: failed waiting for nodepool=\"npdg-4-21\" for cluster \"np-downgrade-tphl5x\" in resourcegroup=\"rg-np-downgrade-tphl5x-crdlsng97qjz\" to finish creating: GET https://rp.j5446400.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/277c2008-b4c1-4e9e-b6d0-fc1bfebe17ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/277c2008-b4c1-4e9e-b6d0-fc1bfebe17ba\",\n \"name\": \"277c2008-b4c1-4e9e-b6d0-fc1bfebe17ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T07:01:20.069431152Z\",\n \"endTime\": \"2026-09-17T07:20:25.508659948Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100463746039156736","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T05:55:56Z","started_at":"2026-09-17T05:55:56Z","completed_at":"2026-09-17T07:58:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:54.351595327Z","timing_metadata_checked_at":"2026-09-18T15:40:54.351595327Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:54.355575742Z","region_metadata_checked_at":"2026-09-18T15:40:54.355575742Z"},"failed_at":["alert"],"failed_test_count":2,"occurrences":[{"row_id":"2b816ec1272ae6ce68eb6a2b25fef38c8808db23627893e729e6d75c44c11b32","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100463746039156736","occurred_at":"2026-09-17T05:55:56Z","failed_at":"alert","signature_id":"b8571782c9062ff08f362067402808930639b058a5f971bedc267ccc71e53617","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T07:19:22Z\n Ended: 2026-09-17T07:25:21Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9156736-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssjt716mpg6m63d08d6sj0snrqfpk2d-oidc-wi-cluster\", pod=\"openshift-apiserver-74544dc599-7mnfx\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssjt716mpg6m63d08d6sj0snrqfpk2d-oidc-wi-cluster/openshift-apiserver-74544dc599-7mnfx has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"c250b3cfce1ecccc3f8e2c8c3420f5f8bc71d156af76632a94dcdfc94649cb73","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100463746039156736","occurred_at":"2026-09-17T05:55:56Z","failed_at":"alert","signature_id":"aaa081b0ec7f2d8d4a7e0e90f29ef2c663585ebb663aa2ffcb4b659e94e0c5f2","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 11 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T07:20:27Z\n Ended: 2026-09-17T07:34:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-677d695895-jblm5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-677d695895-jblm5 (namespace mgmt-agent) on cluster ci01-j9156736-mgmt-1 has averaged 164.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T07:20:27Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-6dd796c54b-tjcwl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-6dd796c54b-tjcwl (namespace aro-hcp) on cluster ci01-j9156736-svc has averaged 404.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T07:20:27Z\n Ended: 2026-09-17T07:34:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fl7w5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fl7w5 (namespace arobit) on cluster ci01-j9156736-mgmt-1 has averaged 172.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T07:20:27Z\n Ended: 2026-09-17T07:44:26Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7d69566948-vrxfz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7d69566948-vrxfz (namespace clusters-service) on cluster ci01-j9156736-svc has averaged 240.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-17T07:20:27Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7d69566948-q5bw5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7d69566948-q5bw5 (namespace clusters-service) on cluster ci01-j9156736-svc has averaged 906.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T07:20:27Z\n Ended: 2026-09-17T07:40:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qpgtp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qpgtp (namespace arobit) on cluster ci01-j9156736-svc has averaged 183% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T07:20:27Z\n Ended: 2026-09-17T07:38:27Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6lzzk\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6lzzk (namespace arobit) on cluster ci01-j9156736-mgmt-2 has averaged 154.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T07:20:27Z\n Ended: 2026-09-17T07:44:26Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7d69566948-gsgx7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7d69566948-gsgx7 (namespace clusters-service) on cluster ci01-j9156736-svc has averaged 244.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Resolved\n Started: 2026-09-17T07:21:27Z\n Ended: 2026-09-17T07:32:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-4sqn2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-4sqn2 (namespace arobit) on cluster ci01-j9156736-mgmt-2 has averaged 128.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T07:23:28Z\n Ended: 2026-09-17T07:31:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-nmkhp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-nmkhp (namespace arobit) on cluster ci01-j9156736-mgmt-1 has averaged 123.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-17T07:24:27Z\n Ended: 2026-09-17T07:34:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156736-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-tbpjs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-tbpjs (namespace arobit) on cluster ci01-j9156736-mgmt-2 has averaged 134.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100445405488615424","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6952,"pr_state":"closed","pr_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","final_merged_sha":"a4de05dadb773673a7d1a63ee2677dbb182bda10","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T04:43:03Z","started_at":"2026-09-17T04:43:03Z","completed_at":"2026-09-17T06:37:16Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:25.685173213Z","timing_metadata_checked_at":"2026-09-18T15:41:25.685173213Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:25.689788129Z","region_metadata_checked_at":"2026-09-18T15:41:25.689788129Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"dd2164b2e6f57e26c14b6a5ac77842577732dc1338d1723de40beaed5461553c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100445405488615424","occurred_at":"2026-09-17T04:43:03Z","failed_at":"alert","signature_id":"98874527ffbf8450aeb4dca36d3c24d835d1bfc8669e736b7a62e3aeb588719f","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 13 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T06:02:43Z\n Ended: 2026-09-17T06:14:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-j5gws\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-j5gws (namespace arobit) on cluster ci01-j8615424-mgmt-1 has averaged 143% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T06:02:43Z\n Ended: 2026-09-17T06:13:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-bmln4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-bmln4 (namespace arobit) on cluster ci01-j8615424-svc has averaged 145.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T06:02:43Z\n Ended: 2026-09-17T06:24:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-fcbf78d68-44rlm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-fcbf78d68-44rlm (namespace clusters-service) on cluster ci01-j8615424-svc has averaged 249.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-17T06:02:43Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-dqfmn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-dqfmn (namespace arobit) on cluster ci01-j8615424-mgmt-2 has averaged 256.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T06:02:43Z\n Ended: 2026-09-17T06:17:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-zsp66\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-zsp66 (namespace arobit) on cluster ci01-j8615424-svc has averaged 172.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T06:02:43Z\n Ended: 2026-09-17T06:23:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-fcbf78d68-fppd7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-fcbf78d68-fppd7 (namespace clusters-service) on cluster ci01-j8615424-svc has averaged 232.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T06:02:43Z\n Ended: 2026-09-17T06:11:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-npwtc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-npwtc (namespace arobit) on cluster ci01-j8615424-mgmt-1 has averaged 211.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T06:02:43Z\n Ended: 2026-09-17T06:14:44Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-f65kz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-f65kz (namespace arobit) on cluster ci01-j8615424-mgmt-1 has averaged 139.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-09-17T06:02:43Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-78dccff4fc-6c5dm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-78dccff4fc-6c5dm (namespace aro-hcp) on cluster ci01-j8615424-svc has averaged 411.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T06:02:43Z\n Ended: 2026-09-17T06:11:45Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5bbcf797b5-tqrgg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5bbcf797b5-tqrgg (namespace mgmt-agent) on cluster ci01-j8615424-mgmt-1 has averaged 214.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Fired (not resolved)\n Started: 2026-09-17T06:02:44Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-fcbf78d68-9x76k\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-fcbf78d68-9x76k (namespace clusters-service) on cluster ci01-j8615424-svc has averaged 920.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n State: Resolved\n Started: 2026-09-17T06:02:44Z\n Ended: 2026-09-17T06:20:46Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5bbcf797b5-jxs79\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5bbcf797b5-jxs79 (namespace mgmt-agent) on cluster ci01-j8615424-mgmt-2 has averaged 196.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 13:\n State: Resolved\n Started: 2026-09-17T06:05:45Z\n Ended: 2026-09-17T06:15:44Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j8615424-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-b9pb4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-b9pb4 (namespace arobit) on cluster ci01-j8615424-mgmt-2 has averaged 130.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100440653564481536","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T04:24:10Z","started_at":"2026-09-17T04:24:10Z","completed_at":"2026-09-17T06:17:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:34.023728795Z","timing_metadata_checked_at":"2026-09-18T15:40:34.023728795Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:34.029666116Z","region_metadata_checked_at":"2026-09-18T15:40:34.029666116Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"1c1cdc305c0c7ff6da6b0da330d34cb537929c0a428231693509a3829a05de5d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100440653564481536","occurred_at":"2026-09-17T04:24:10Z","failed_at":"alert","signature_id":"98473ea7cdc506ea6ed06c90a329992858dfe045473eb058bcb45937743f020e","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T05:19:56Z\n Ended: 2026-09-17T05:52:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4481536-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6nhnd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6nhnd (namespace arobit) on cluster ci01-j4481536-svc has averaged 193.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T05:21:54Z\n Ended: 2026-09-17T06:05:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4481536-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5bd75f6d87-6frgs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5bd75f6d87-6frgs (namespace clusters-service) on cluster ci01-j4481536-svc has averaged 671.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T05:21:54Z\n Ended: 2026-09-17T05:58:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4481536-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-864cf4d9dd-5gkfh\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-864cf4d9dd-5gkfh (namespace aro-hcp) on cluster ci01-j4481536-svc has averaged 380.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T05:22:57Z\n Ended: 2026-09-17T05:35:57Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4481536-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5bd75f6d87-75jjn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5bd75f6d87-75jjn (namespace clusters-service) on cluster ci01-j4481536-svc has averaged 233.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T05:22:57Z\n Ended: 2026-09-17T05:38:53Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4481536-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5bd75f6d87-w2tsd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5bd75f6d87-w2tsd (namespace clusters-service) on cluster ci01-j4481536-svc has averaged 251.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T05:26:55Z\n Ended: 2026-09-17T05:45:54Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4481536-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vgw8g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-vgw8g (namespace arobit) on cluster ci01-j4481536-mgmt-2 has averaged 140.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6986,"pr_state":"closed","pr_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","final_merged_sha":"86284f63a02ed834c996c3e85c3fed26fe0ed27c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T04:14:58Z","started_at":"2026-09-17T04:14:58Z","completed_at":"2026-09-17T06:06:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:09.889629369Z","timing_metadata_checked_at":"2026-09-18T15:41:09.889629369Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:09.893965285Z","region_metadata_checked_at":"2026-09-18T15:41:09.893965285Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"3992b9b92791366ae58ddfb857a32e27369393ac8e1628f51c0c526849e2621e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","failed_at":"alert","signature_id":"d43422fbdada1a940ecd84da27859063da487a9ed357cd0dce3b03a5e1093c47","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 3 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T05:25:34Z\n Ended: 2026-09-17T05:33:33Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5021696-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5765fc97bc-hmsgv\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5765fc97bc-hmsgv (namespace clusters-service) on cluster ci01-j5021696-svc has averaged 229.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T05:25:34Z\n Ended: 2026-09-17T05:34:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5021696-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5765fc97bc-7rqzw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5765fc97bc-7rqzw (namespace clusters-service) on cluster ci01-j5021696-svc has averaged 785% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T05:28:32Z\n Ended: 2026-09-17T05:34:32Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5021696-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-dbnnd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-dbnnd (namespace arobit) on cluster ci01-j5021696-svc has averaged 126.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"5ba641e8371bba3b667de99b268b7a7877d7b51de4c9a9069563b457ebd96b0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","failed_at":"e2e","signature_id":"421f5b6d838c121843fe235c14d4d70af34427ca212e0849c68ed8aa29654d6b","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc120\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w5shsm9ng8kc\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n \"name\": \"eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:52:38.625826904Z\",\n \"endTime\": \"2026-09-17T05:21:45.494759028Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 2m17s ago; version 4.22.14 is Partial (want Completed), started 5m22s ago; version 4.22.14 is Partial (want Completed), started 12m16s ago; version 4.22.14 is Partial (want Completed), started 15m20s ago; version 4.22.14 is Partial (want Completed), started 17m23s ago; version 4.22.14 is Partial (want Completed), started 17m58s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc120\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w5shsm9ng8kc\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n \"name\": \"eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:52:38.625826904Z\",\n \"endTime\": \"2026-09-17T05:21:45.494759028Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 2m17s ago; version 4.22.14 is Partial (want Completed), started 5m22s ago; version 4.22.14 is Partial (want Completed), started 12m16s ago; version 4.22.14 is Partial (want Completed), started 15m20s ago; version 4.22.14 is Partial (want Completed), started 17m23s ago; version 4.22.14 is Partial (want Completed), started 17m58s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"46425ffc9312d96c5106d1b27946c7775e4ddf6d0471e792a3aaf8c2bc4d3a86","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb10ba3c02c74852c8b1cfe4309b1aa865949371b662245fea95cd7fa071f301","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","failed_at":"e2e","signature_id":"82d6e28574c7f34a07c333ca94f997752b96649f6b80a188942914d02804a6b8","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010be220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-r2cp7qnm68cd\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1574465a-b85d-4740-b4e8-67565d3ad14a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1574465a-b85d-4740-b4e8-67565d3ad14a\",\n \"name\": \"1574465a-b85d-4740-b4e8-67565d3ad14a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:51:44.941548523Z\",\n \"endTime\": \"2026-09-17T05:20:45.403813623Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:85]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010be220\u003e: \n failed to create HCP cluster pause-bkp-cluster: failed waiting for cluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-r2cp7qnm68cd\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1574465a-b85d-4740-b4e8-67565d3ad14a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1574465a-b85d-4740-b4e8-67565d3ad14a\",\n \"name\": \"1574465a-b85d-4740-b4e8-67565d3ad14a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:51:44.941548523Z\",\n \"endTime\": \"2026-09-17T05:20:45.403813623Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m27s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","46425ffc9312d96c5106d1b27946c7775e4ddf6d0471e792a3aaf8c2bc4d3a86","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6917,"pr_state":"open","pr_sha":"55efd56d7b76c88f48195a55ad9df67aaf1f0f91","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T04:05:17Z","started_at":"2026-09-17T04:05:17Z","completed_at":"2026-09-17T05:58:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:39.966836578Z","timing_metadata_checked_at":"2026-09-18T15:40:39.966836578Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:39.97031399Z","region_metadata_checked_at":"2026-09-18T15:40:39.97031399Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"7130f6d51bdd624d030677b89465270ae8acdd4503ab8b66831172bd5acd6eb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","failed_at":"alert","signature_id":"a15a05f0f8abcad7e64ec9115ec84f6ac3e540a459d0e285796dd856a7a6caa3","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T05:23:48Z\n Ended: 2026-09-17T05:29:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4260864-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2vs49\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-2vs49 (namespace arobit) on cluster ci01-j4260864-mgmt-2 has averaged 172.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T05:26:47Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4260864-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-b498k\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-b498k (namespace arobit) on cluster ci01-j4260864-mgmt-1 has averaged 202.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"def12c5406c87b9df3eb15c89cc11b427357c2143e13615c3b3cddebfcf91e55","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","failed_at":"e2e","signature_id":"2938ba362ea03badfa4fef3a5a8b455d8db3137be286b0a32f0e06cb64bc56e1","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-lkx9zk2x2sg4/cluster-candidate-4-22-v22kq5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001068640\u003e: \n failed to create HCP cluster cluster-candidate-4-22-v22kq5: failed waiting for cluster=\"cluster-candidate-4-22-v22kq5\" in resourcegroup=\"rg-candidate-4-22-lkx9zk2x2sg4\" to finish creating: GET https://rp.j4260864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/741d8f04-ed8d-4645-94fa-56f03c47eb8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/741d8f04-ed8d-4645-94fa-56f03c47eb8f\",\n \"name\": \"741d8f04-ed8d-4645-94fa-56f03c47eb8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:52:59.588772018Z\",\n \"endTime\": \"2026-09-17T05:21:59.743565867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-lkx9zk2x2sg4/cluster-candidate-4-22-v22kq5 should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001068640\u003e: \n failed to create HCP cluster cluster-candidate-4-22-v22kq5: failed waiting for cluster=\"cluster-candidate-4-22-v22kq5\" in resourcegroup=\"rg-candidate-4-22-lkx9zk2x2sg4\" to finish creating: GET https://rp.j4260864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/741d8f04-ed8d-4645-94fa-56f03c47eb8f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/741d8f04-ed8d-4645-94fa-56f03c47eb8f\",\n \"name\": \"741d8f04-ed8d-4645-94fa-56f03c47eb8f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:52:59.588772018Z\",\n \"endTime\": \"2026-09-17T05:21:59.743565867Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 23m13s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e13c765a12c2a8b62bd544dd688205dc0d68584a8089d89e1b89352d0cc724d5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","failed_at":"e2e","signature_id":"89c35af4f14189b65c64576cd7490a0088f1c8ab485b614bf43cd6ed4c19e683","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca6240\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-drg9bd4zkzmf\" to finish creating: GET https://rp.j4260864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/53c7201f-3e66-49f4-8e9d-3ea9fed2dbcb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/53c7201f-3e66-49f4-8e9d-3ea9fed2dbcb\",\n \"name\": \"53c7201f-3e66-49f4-8e9d-3ea9fed2dbcb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:51:59.38221738Z\",\n \"endTime\": \"2026-09-17T05:20:59.747749137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ca6240\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-drg9bd4zkzmf\" to finish creating: GET https://rp.j4260864.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/53c7201f-3e66-49f4-8e9d-3ea9fed2dbcb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/53c7201f-3e66-49f4-8e9d-3ea9fed2dbcb\",\n \"name\": \"53c7201f-3e66-49f4-8e9d-3ea9fed2dbcb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:51:59.38221738Z\",\n \"endTime\": \"2026-09-17T05:20:59.747749137Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m58s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100434265043898368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T03:58:47Z","started_at":"2026-09-17T03:58:47Z","completed_at":"2026-09-17T05:53:55Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:54.222423463Z","timing_metadata_checked_at":"2026-09-18T15:40:54.222423463Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:54.226473977Z","region_metadata_checked_at":"2026-09-18T15:40:54.226473977Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0c70b3c8e07e53a189d09765b019a9c34d5ad0bb5785af4fa5f30ac1ca00f3ad","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100434265043898368","occurred_at":"2026-09-17T03:58:47Z","failed_at":"alert","signature_id":"3e9ecaaa55541e660e55a54f116bbcb93077443ad73a39702bb7f87aa56a43f5","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T05:21:49Z\n Ended: 2026-09-17T05:28:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3898368-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6456884798-89rmr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-6456884798-89rmr (namespace mgmt-agent) on cluster ci01-j3898368-mgmt-2 has averaged 142.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T05:21:49Z\n Ended: 2026-09-17T05:28:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3898368-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5957d786fb-xjs47\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5957d786fb-xjs47 (namespace clusters-service) on cluster ci01-j3898368-svc has averaged 978.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T05:21:49Z\n Ended: 2026-09-17T05:36:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3898368-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-r86jf\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-r86jf (namespace arobit) on cluster ci01-j3898368-mgmt-1 has averaged 194% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T05:21:49Z\n Ended: 2026-09-17T05:29:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3898368-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-nbqjq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-nbqjq (namespace arobit) on cluster ci01-j3898368-svc has averaged 289.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T05:21:49Z\n Ended: 2026-09-17T05:36:50Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3898368-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6456884798-h72vz\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-6456884798-h72vz (namespace mgmt-agent) on cluster ci01-j3898368-mgmt-1 has averaged 193.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T05:21:49Z\n Ended: 2026-09-17T05:44:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3898368-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5957d786fb-cz742\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5957d786fb-cz742 (namespace clusters-service) on cluster ci01-j3898368-svc has averaged 254.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T05:21:49Z\n Ended: 2026-09-17T05:28:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3898368-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-d6dd4cc9f-qs4kx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-d6dd4cc9f-qs4kx (namespace aro-hcp) on cluster ci01-j3898368-svc has averaged 491.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T05:21:49Z\n Ended: 2026-09-17T05:28:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3898368-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-dvnvp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-dvnvp (namespace arobit) on cluster ci01-j3898368-mgmt-2 has averaged 141.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"b174572dc191a608e507b7908c306f563d468984a6a806ca9457937a96a7c811","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100434265043898368","occurred_at":"2026-09-17T03:58:47Z","failed_at":"e2e","signature_id":"297e17fd0f8b231f49a6a64a53967d8a3faf0348a5fdc0a0860997681f536656","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0014180c0\u003e: \n VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-phbtk2 namespace are ready: failed to list pods in cilium-connectivity-check-phbtk2 namespace: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc0014180c0\u003e: \n VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-phbtk2 namespace are ready: failed to list pods in cilium-connectivity-check-phbtk2 namespace: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T03:04:31Z","started_at":"2026-09-17T03:04:31Z","completed_at":"2026-09-17T04:42:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:25.507927073Z","timing_metadata_checked_at":"2026-09-18T15:41:25.507927073Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:25.512821491Z","region_metadata_checked_at":"2026-09-18T15:41:25.512821491Z"},"failed_at":["alert","e2e"],"failed_test_count":3,"occurrences":[{"row_id":"6807b896e16e806b9f74601ef9e11a7f92534c90e8d074fbca2abe44644cb617","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","failed_at":"alert","signature_id":"8469f16144ba929df5b9d7bb8ddc370454c226da3349d92d91c2c32c02a33ba7","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T04:08:23Z\n Ended: 2026-09-17T04:14:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0741376-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5b859bc4bf-jjt2z\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5b859bc4bf-jjt2z (namespace clusters-service) on cluster ci01-j0741376-svc has averaged 832% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T04:08:23Z\n Ended: 2026-09-17T04:16:23Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0741376-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5d4d9cb45c-7rb4g\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5d4d9cb45c-7rb4g (namespace mgmt-agent) on cluster ci01-j0741376-mgmt-2 has averaged 168.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T04:08:23Z\n Ended: 2026-09-17T04:19:25Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0741376-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-c9l25\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-c9l25 (namespace arobit) on cluster ci01-j0741376-mgmt-2 has averaged 140.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T04:08:23Z\n Ended: 2026-09-17T04:22:26Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0741376-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-84nsd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-84nsd (namespace arobit) on cluster ci01-j0741376-mgmt-2 has averaged 174.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T04:14:27Z\n Ended: 2026-09-17T04:21:23Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0741376-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-v9grg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-v9grg (namespace arobit) on cluster ci01-j0741376-mgmt-2 has averaged 123.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"86132afabf8cd9ebe3670b18e63499850df00600ba6ecabe248a57e533dfe636","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","failed_at":"e2e","signature_id":"72e9dcaaa03f26fa26de2141ffb6133bfeb9f62e98e7bbfc149c580a12c625ab","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-mvfnshj95l4p/cluster-candidate-5-0-9ptr9v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010149a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9ptr9v: failed waiting for cluster=\"cluster-candidate-5-0-9ptr9v\" in resourcegroup=\"rg-candidate-5-0-mvfnshj95l4p\" to finish creating: GET https://rp.j0741376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f558a3f-8e1b-4977-a6b0-a4b70992d9bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f558a3f-8e1b-4977-a6b0-a4b70992d9bb\",\n \"name\": \"2f558a3f-8e1b-4977-a6b0-a4b70992d9bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T03:35:00.650591919Z\",\n \"endTime\": \"2026-09-17T04:04:02.019280064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 21m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-5-0-mvfnshj95l4p/cluster-candidate-5-0-9ptr9v should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010149a0\u003e: \n failed to create HCP cluster cluster-candidate-5-0-9ptr9v: failed waiting for cluster=\"cluster-candidate-5-0-9ptr9v\" in resourcegroup=\"rg-candidate-5-0-mvfnshj95l4p\" to finish creating: GET https://rp.j0741376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f558a3f-8e1b-4977-a6b0-a4b70992d9bb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f558a3f-8e1b-4977-a6b0-a4b70992d9bb\",\n \"name\": \"2f558a3f-8e1b-4977-a6b0-a4b70992d9bb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T03:35:00.650591919Z\",\n \"endTime\": \"2026-09-17T04:04:02.019280064Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.2 is Partial (want Completed), started 21m52s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f039978ef296066211e4d4aa2527d6df183f09e7a25fa4df1ff6b091ba1f75b8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","failed_at":"e2e","signature_id":"4b99cbe35f0bd3c6197a97e4f313692f0f80c116d5fba8a8d582ebf622976c32","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8f5e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-n2zmdkblldlw\" to finish creating: GET https://rp.j0741376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cfa413af-9795-4d14-a521-6638502432ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cfa413af-9795-4d14-a521-6638502432ab\",\n \"name\": \"cfa413af-9795-4d14-a521-6638502432ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:05:39.31416638Z\",\n \"endTime\": \"2026-09-17T04:06:12.119697969Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b8f5e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-n2zmdkblldlw\" to finish creating: GET https://rp.j0741376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cfa413af-9795-4d14-a521-6638502432ab\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cfa413af-9795-4d14-a521-6638502432ab\",\n \"name\": \"cfa413af-9795-4d14-a521-6638502432ab\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T04:05:39.31416638Z\",\n \"endTime\": \"2026-09-17T04:06:12.119697969Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100415215534346240","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6439,"pr_state":"closed","pr_sha":"b6b00cdc8ea3ee249a9faa811955e4eb095652ef","final_merged_sha":"8e51fd56182fb53110559c6686373da1a4435f34","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T02:43:05Z","started_at":"2026-09-17T02:43:05Z","completed_at":"2026-09-17T05:18:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:06.527082263Z","timing_metadata_checked_at":"2026-09-18T15:40:06.527082263Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:06.530229874Z","region_metadata_checked_at":"2026-09-18T15:40:06.530229874Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"959a00d1b6da0a4969ad502ef2a652b53058dabc03fd7ac37b587e345be30afa","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100415215534346240","occurred_at":"2026-09-17T02:43:05Z","failed_at":"alert","signature_id":"05d61646d01952c54a7da4864113e0532f1dbe45108ad4402418ae2eaaf5a2b6","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T04:31:05Z\n Ended: 2026-09-17T04:49:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4346240-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fjvpp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fjvpp (namespace arobit) on cluster ci01-j4346240-svc has averaged 164.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T04:31:05Z\n Ended: 2026-09-17T04:49:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4346240-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-57c8fd9876-xlstr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-57c8fd9876-xlstr (namespace aro-hcp) on cluster ci01-j4346240-svc has averaged 388.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T04:32:03Z\n Ended: 2026-09-17T04:49:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4346240-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-lv9vn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-lv9vn (namespace arobit) on cluster ci01-j4346240-mgmt-1 has averaged 143.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T04:32:03Z\n Ended: 2026-09-17T04:41:03Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4346240-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5b7c47c974-6qcss\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5b7c47c974-6qcss (namespace clusters-service) on cluster ci01-j4346240-svc has averaged 896.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T04:34:06Z\n Ended: 2026-09-17T04:41:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4346240-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-klvzm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-klvzm (namespace arobit) on cluster ci01-j4346240-svc has averaged 138.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T04:34:07Z\n Ended: 2026-09-17T04:48:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4346240-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5b7c47c974-8jpp2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5b7c47c974-8jpp2 (namespace clusters-service) on cluster ci01-j4346240-svc has averaged 221.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T04:34:07Z\n Ended: 2026-09-17T04:52:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4346240-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-6bf86fbcd6-47cmn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-6bf86fbcd6-47cmn (namespace mgmt-agent) on cluster ci01-j4346240-mgmt-1 has averaged 176.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T04:43:05Z\n Ended: 2026-09-17T04:50:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j4346240-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-gvr9v\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-gvr9v (namespace arobit) on cluster ci01-j4346240-mgmt-2 has averaged 129% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100404999925272576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6980,"pr_state":"closed","pr_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","final_merged_sha":"4c6ef86c4410d1e48dd7653c177a4370cf38a246","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T02:02:30Z","started_at":"2026-09-17T02:02:30Z","completed_at":"2026-09-17T04:55:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:08.064731581Z","timing_metadata_checked_at":"2026-09-18T15:41:08.064731581Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:08.070365001Z","region_metadata_checked_at":"2026-09-18T15:41:08.070365001Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"790f368718a2112337169a3c91cdffa186c4b97f083f3f664e7d584b843c20c0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100404999925272576","occurred_at":"2026-09-17T02:02:30Z","failed_at":"e2e","signature_id":"17d592d037b2f2a804c6494b323dd9c8826c7ee66667ea66a41a04ee3ac208fe","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-4jpl9dtwpfzm/cluster-candidate-4-22-shq9rw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007f6520\u003e: \n failed to create HCP cluster cluster-candidate-4-22-shq9rw: failed waiting for cluster=\"cluster-candidate-4-22-shq9rw\" in resourcegroup=\"rg-candidate-4-22-4jpl9dtwpfzm\" to finish creating: GET https://rp.j5272576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f2fdee1-3fa0-4cad-b40d-396308bf0040\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f2fdee1-3fa0-4cad-b40d-396308bf0040\",\n \"name\": \"8f2fdee1-3fa0-4cad-b40d-396308bf0040\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T03:44:24.882818903Z\",\n \"endTime\": \"2026-09-17T04:13:25.759327578Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-4jpl9dtwpfzm/cluster-candidate-4-22-shq9rw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007f6520\u003e: \n failed to create HCP cluster cluster-candidate-4-22-shq9rw: failed waiting for cluster=\"cluster-candidate-4-22-shq9rw\" in resourcegroup=\"rg-candidate-4-22-4jpl9dtwpfzm\" to finish creating: GET https://rp.j5272576.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f2fdee1-3fa0-4cad-b40d-396308bf0040\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/8f2fdee1-3fa0-4cad-b40d-396308bf0040\",\n \"name\": \"8f2fdee1-3fa0-4cad-b40d-396308bf0040\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T03:44:24.882818903Z\",\n \"endTime\": \"2026-09-17T04:13:25.759327578Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 22m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8bdf60d92a210c031b38af1009211605d99488b68f172e1efc7b26bdeb3f88ef","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100404999925272576","occurred_at":"2026-09-17T02:02:30Z","failed_at":"alert","signature_id":"030419f084e0ac9f8645ed83bbb93e54a5eee29ea545c7c537285ba77f7a19f7","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 15 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T04:13:05Z\n Ended: 2026-09-17T04:32:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-lbvfd\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-lbvfd (namespace arobit) on cluster ci01-j5272576-mgmt-1 has averaged 158% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T04:13:05Z\n Ended: 2026-09-17T04:34:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-djr26\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-djr26 (namespace arobit) on cluster ci01-j5272576-svc has averaged 183.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-17T04:14:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7f68fcd5b9-456ls\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7f68fcd5b9-456ls (namespace clusters-service) on cluster ci01-j5272576-svc has averaged 831.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T04:14:06Z\n Ended: 2026-09-17T04:44:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-b6fc9ffcf-w7hl2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-b6fc9ffcf-w7hl2 (namespace aro-hcp) on cluster ci01-j5272576-svc has averaged 409% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T04:14:06Z\n Ended: 2026-09-17T04:41:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7f68fcd5b9-7mv4x\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7f68fcd5b9-7mv4x (namespace clusters-service) on cluster ci01-j5272576-svc has averaged 271.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T04:14:06Z\n Ended: 2026-09-17T04:40:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-n9r9l\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-n9r9l (namespace arobit) on cluster ci01-j5272576-mgmt-2 has averaged 162.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T04:14:06Z\n Ended: 2026-09-17T04:37:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-55wzp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-55wzp (namespace arobit) on cluster ci01-j5272576-mgmt-2 has averaged 196.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T04:14:06Z\n Ended: 2026-09-17T04:38:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-7f68fcd5b9-d8nlf\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-7f68fcd5b9-d8nlf (namespace clusters-service) on cluster ci01-j5272576-svc has averaged 233.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Resolved\n Started: 2026-09-17T04:14:06Z\n Ended: 2026-09-17T04:28:07Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-q6c4s\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-q6c4s (namespace arobit) on cluster ci01-j5272576-mgmt-1 has averaged 137.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T04:15:04Z\n Ended: 2026-09-17T04:32:03Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-lj9x9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-lj9x9 (namespace arobit) on cluster ci01-j5272576-mgmt-1 has averaged 145.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-17T04:15:04Z\n Ended: 2026-09-17T04:29:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8k7gj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8k7gj (namespace arobit) on cluster ci01-j5272576-svc has averaged 138.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n State: Resolved\n Started: 2026-09-17T04:15:04Z\n Ended: 2026-09-17T04:38:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-gj7rn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-gj7rn (namespace arobit) on cluster ci01-j5272576-mgmt-2 has averaged 161.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 13:\n State: Resolved\n Started: 2026-09-17T04:15:04Z\n Ended: 2026-09-17T04:33:05Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-r2j2n\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-r2j2n (namespace arobit) on cluster ci01-j5272576-mgmt-2 has averaged 141.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 14:\n State: Resolved\n Started: 2026-09-17T04:16:05Z\n Ended: 2026-09-17T04:30:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5b8f86b79b-jv7v4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5b8f86b79b-jv7v4 (namespace mgmt-agent) on cluster ci01-j5272576-mgmt-1 has averaged 161.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 15:\n State: Resolved\n Started: 2026-09-17T04:16:05Z\n Ended: 2026-09-17T04:33:06Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j5272576-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5b8f86b79b-7qs69\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5b8f86b79b-7qs69 (namespace mgmt-agent) on cluster ci01-j5272576-mgmt-2 has averaged 187% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6976,"pr_state":"open","pr_sha":"3f37770fba22a55b4336e3a85cbf275bf37c8279","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T01:53:32Z","started_at":"2026-09-17T01:53:32Z","completed_at":"2026-09-17T03:47:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:05.341236174Z","timing_metadata_checked_at":"2026-09-18T15:41:05.341236174Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:05.344249885Z","region_metadata_checked_at":"2026-09-18T15:41:05.344249885Z"},"failed_at":["alert","e2e"],"failed_test_count":6,"occurrences":[{"row_id":"21398ac16c4cf04338f000da8c64fcba89f74045be9cb5a6da725fed9e266ecf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","failed_at":"e2e","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"29ebcac02ea8fbbb5605db5d6f832ab98e481b0a22ab484822f70098dafe0a24","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","failed_at":"alert","signature_id":"082cd49a16ae701dbb38b1b79f0d1ac12e5a9df37271b1daf757d49fb6b996f0","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T02:45:46Z\n Ended: 2026-09-17T03:32:46Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssg777qqnq4cjon6gmqfatr43murnr3-v8z9h0b9l2l6b7a\", pod=\"router-6777b8c95f-lklkr\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssg777qqnq4cjon6gmqfatr43murnr3-v8z9h0b9l2l6b7a/router-6777b8c95f-lklkr has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T02:45:46Z\n Ended: 2026-09-17T03:25:44Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssg777qqnq4cjon6gmqfatr43murnr3-v8z9h0b9l2l6b7a\", pod=\"router-6777b8c95f-kwgxc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssg777qqnq4cjon6gmqfatr43murnr3-v8z9h0b9l2l6b7a/router-6777b8c95f-kwgxc has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T02:46:47Z\n Ended: 2026-09-17T03:04:42Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssg7mbkri4692gdg08dld9mm65i0ctd-w7q9x5r4e9d1u2u\", pod=\"router-d6d8587c6-stnlp\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssg7mbkri4692gdg08dld9mm65i0ctd-w7q9x5r4e9d1u2u/router-d6d8587c6-stnlp has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T02:48:43Z\n Ended: 2026-09-17T03:04:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssg85jl4ed13fvm3rmk51oupprdb08b-j1m7o6h4e6k8x0m\", pod=\"router-7cd4947c5d-fcvgq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssg85jl4ed13fvm3rmk51oupprdb08b-j1m7o6h4e6k8x0m/router-7cd4947c5d-fcvgq has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-17T02:48:43Z\n Ended: 2026-09-17T03:04:43Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssg8gg615bvgkkgo2s0arenrfb5gltm-agg-cluster\", pod=\"router-866b68f94b-wr5tc\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssg8gg615bvgkkgo2s0arenrfb5gltm-agg-cluster/router-866b68f94b-wr5tc has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"504ed237a2614eb636d2d76d05ca0dc93dfa34c82fbdb9c9b7f31118c1f769b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","failed_at":"alert","signature_id":"09d8e2ae3fbdc5cec5986413598d96675056845bffdde0b6a135d4a6c2fcafa1","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 11 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T03:10:49Z\n Ended: 2026-09-17T03:28:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vv2pt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-vv2pt (namespace arobit) on cluster ci01-j9076992-mgmt-2 has averaged 161.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T03:10:49Z\n Ended: 2026-09-17T03:25:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qm88r\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qm88r (namespace arobit) on cluster ci01-j9076992-mgmt-2 has averaged 178.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-17T03:10:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-56459cc9f-dddn4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-56459cc9f-dddn4 (namespace clusters-service) on cluster ci01-j9076992-svc has averaged 831.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Fired (not resolved)\n Started: 2026-09-17T03:10:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-75748f55df-tzjvh\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-75748f55df-tzjvh (namespace aro-hcp) on cluster ci01-j9076992-svc has averaged 422.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T03:10:49Z\n Ended: 2026-09-17T03:22:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-76df9b9f8-lm9mq\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-76df9b9f8-lm9mq (namespace mgmt-agent) on cluster ci01-j9076992-mgmt-1 has averaged 168.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T03:10:49Z\n Ended: 2026-09-17T03:33:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-56459cc9f-vsgg5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-56459cc9f-vsgg5 (namespace clusters-service) on cluster ci01-j9076992-svc has averaged 254.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T03:10:49Z\n Ended: 2026-09-17T03:20:49Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2dtcx\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-2dtcx (namespace arobit) on cluster ci01-j9076992-mgmt-1 has averaged 147.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T03:10:49Z\n Ended: 2026-09-17T03:31:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rrbhs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-rrbhs (namespace arobit) on cluster ci01-j9076992-svc has averaged 235.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Resolved\n Started: 2026-09-17T03:10:49Z\n Ended: 2026-09-17T03:27:52Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-76df9b9f8-qhgv5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-76df9b9f8-qhgv5 (namespace mgmt-agent) on cluster ci01-j9076992-mgmt-2 has averaged 229.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T03:10:49Z\n Ended: 2026-09-17T03:31:48Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-mps8r\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-mps8r (namespace arobit) on cluster ci01-j9076992-mgmt-2 has averaged 267.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-17T03:10:49Z\n Ended: 2026-09-17T03:24:51Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9076992-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6znl8\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6znl8 (namespace arobit) on cluster ci01-j9076992-mgmt-1 has averaged 155.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"6e8de28554689f6c3e24af63d9bbce5bc7b00c1cc20e584d4c400dc504dee0a2","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","failed_at":"e2e","signature_id":"2bd939aaad187ab78804671b6970c3fb300c0b1bc1be22955c06bc837de68bb9","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00168f0c8\u003e: \n VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-txdwxr namespace are ready: failed to list pods in cilium-connectivity-check-txdwxr namespace: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n \u003c*errors.joinError | 0xc00168f0c8\u003e: \n VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-txdwxr namespace are ready: failed to list pods in cilium-connectivity-check-txdwxr namespace: client rate limiter Wait returned an error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded"}},{"row_id":"819d7669fb88617d9d39e6a86008d925282ca32af28f9e7baea94518dbbff0c1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","failed_at":"alert","signature_id":"6133b883dec1a7ae7199f6897e93bfe8938d4615907efd1f6e96f8f531de45b7","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T03:10:45Z\n Ended: 2026-09-17T03:32:50Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/hcps-j9076992\", namespace=\"ocm-arohcpci01-2ssg777qqnq4cjon6gmqfatr43murnr3-v8z9h0b9l2l6b7a\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ssg777qqnq4cjon6gmqfatr43murnr3-v8z9h0b9l2l6b7a/router has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}},{"row_id":"d120f397ce50d24046695b247ff91fe69a21192e1b63a09390626a48cfd9f56b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","failed_at":"alert","signature_id":"69b255fd750c54b7a1baf2eca24627dd8e61f4fdef893acd92470a12832db0f6","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 5 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T03:10:30Z\n Ended: 2026-09-17T03:22:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T03:10:30Z\n Ended: 2026-09-17T03:22:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T03:10:30Z\n Ended: 2026-09-17T03:22:31Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T03:11:28Z\n Ended: 2026-09-17T03:21:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-17T03:11:28Z\n Ended: 2026-09-17T03:21:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":6,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875"],"matched_occurrences":6,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100399662610518016","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6009,"pr_state":"open","pr_sha":"758965d1a66d0df407f9e4cf3b6ed6d71c530aad","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T01:41:17Z","started_at":"2026-09-17T01:41:17Z","completed_at":"2026-09-17T04:20:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:04.381834639Z","timing_metadata_checked_at":"2026-09-18T15:40:04.381834639Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:04.38500235Z","region_metadata_checked_at":"2026-09-18T15:40:04.38500235Z"},"failed_at":["alert"],"failed_test_count":1,"occurrences":[{"row_id":"4a0dc86f2c35c684f8f5706ff610c75b6b3f4132ad78cee8c7c9c5e4898f5619","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100399662610518016","occurred_at":"2026-09-17T01:41:17Z","failed_at":"alert","signature_id":"5a274f8276581e7a1744e1e3283d6f882212c3ec0a22900f40bdbce973372f71","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 6 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T03:36:01Z\n Ended: 2026-09-17T03:45:59Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0518016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5b88c9f94f-r8b5t\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5b88c9f94f-r8b5t (namespace clusters-service) on cluster ci01-j0518016-svc has averaged 746.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T03:36:01Z\n Ended: 2026-09-17T03:45:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0518016-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-5b88c9f94f-vh26x\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-5b88c9f94f-vh26x (namespace clusters-service) on cluster ci01-j0518016-svc has averaged 209% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T03:36:01Z\n Ended: 2026-09-17T03:43:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0518016-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-krvq7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-krvq7 (namespace arobit) on cluster ci01-j0518016-mgmt-2 has averaged 218% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T03:36:01Z\n Ended: 2026-09-17T03:43:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0518016-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5485957bf9-d5zv5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5485957bf9-d5zv5 (namespace mgmt-agent) on cluster ci01-j0518016-mgmt-2 has averaged 171.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T03:36:02Z\n Ended: 2026-09-17T03:45:59Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0518016-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5485957bf9-sl6nr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5485957bf9-sl6nr (namespace mgmt-agent) on cluster ci01-j0518016-mgmt-1 has averaged 160.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T03:42:01Z\n Ended: 2026-09-17T03:51:04Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0518016-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-55f47\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-55f47 (namespace arobit) on cluster ci01-j0518016-mgmt-2 has averaged 130.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6940,"pr_state":"closed","pr_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","final_merged_sha":"62cdd448d25099f3b77adb5f5e21439e857c6d5f","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T00:56:54Z","started_at":"2026-09-17T00:56:54Z","completed_at":"2026-09-17T03:16:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:46.467548371Z","timing_metadata_checked_at":"2026-09-18T15:40:46.467548371Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:46.470569081Z","region_metadata_checked_at":"2026-09-18T15:40:46.470569081Z"},"failed_at":["alert","e2e"],"failed_test_count":34,"occurrences":[{"row_id":"02eba019d8e0352c6a617110bdf72ac4dba4e633c89308e6b76c1a2cad5ce4c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"alert","signature_id":"5eff94552133a60d52e39851d75edd9d10579e18460fb3418389d4051065a47d","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 7 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T02:38:15Z\n Ended: 2026-09-17T03:00:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7881088-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-t9b25\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-t9b25 (namespace arobit) on cluster ci01-j7881088-svc has averaged 218.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T02:38:15Z\n Ended: 2026-09-17T03:01:19Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7881088-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-db48c88df-mqr6n\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-db48c88df-mqr6n (namespace clusters-service) on cluster ci01-j7881088-svc has averaged 233% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T02:38:15Z\n Ended: 2026-09-17T02:56:14Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7881088-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-ph5tc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-ph5tc (namespace arobit) on cluster ci01-j7881088-mgmt-2 has averaged 159.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T02:38:15Z\n Ended: 2026-09-17T02:58:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7881088-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-6gsds\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-6gsds (namespace arobit) on cluster ci01-j7881088-mgmt-2 has averaged 188% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-17T02:38:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7881088-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-db48c88df-nsd25\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-db48c88df-nsd25 (namespace clusters-service) on cluster ci01-j7881088-svc has averaged 766.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T02:38:15Z\n Ended: 2026-09-17T02:47:15Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7881088-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-sdthl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-sdthl (namespace arobit) on cluster ci01-j7881088-mgmt-1 has averaged 234.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T02:38:15Z\n Ended: 2026-09-17T03:01:19Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j7881088-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-db48c88df-mbwff\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-db48c88df-mbwff (namespace clusters-service) on cluster ci01-j7881088-svc has averaged 251.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"0471783b478e23d19d505b9b68e6a0b1089f8e76868ab5f269ba447b3f068ce7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"671e3e4773a45e7910b33f7d44f7a3f0d6be8188410cc6c10ced0d14da59716b","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001340300\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jvzc8h86q97v\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1730483c-5a34-4a74-b761-00a13efee4d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1730483c-5a34-4a74-b761-00a13efee4d4\",\n \"name\": \"1730483c-5a34-4a74-b761-00a13efee4d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.276946332Z\",\n \"endTime\": \"2026-09-17T02:31:34.003715435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001340300\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-jvzc8h86q97v\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1730483c-5a34-4a74-b761-00a13efee4d4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1730483c-5a34-4a74-b761-00a13efee4d4\",\n \"name\": \"1730483c-5a34-4a74-b761-00a13efee4d4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.276946332Z\",\n \"endTime\": \"2026-09-17T02:31:34.003715435Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"060cb51bf0762f2fef44be6e28ffb1577f639fa1af25b26ba6a1dcafa932f6b3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"a8a2c2adbed4b3b26d51343db3e33c8589838c2cbac6661fcee12fbe4bf879fe","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118e660\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-9gxmxx6fhffd\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a242de7a-6d2e-430d-bb11-b7c9eaee8899\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a242de7a-6d2e-430d-bb11-b7c9eaee8899\",\n \"name\": \"a242de7a-6d2e-430d-bb11-b7c9eaee8899\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.338798634Z\",\n \"endTime\": \"2026-09-17T02:31:33.973889478Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00118e660\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-9gxmxx6fhffd\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a242de7a-6d2e-430d-bb11-b7c9eaee8899\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a242de7a-6d2e-430d-bb11-b7c9eaee8899\",\n \"name\": \"a242de7a-6d2e-430d-bb11-b7c9eaee8899\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.338798634Z\",\n \"endTime\": \"2026-09-17T02:31:33.973889478Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"09948e0e7d6483fc2a1de030ede651ea348641914dfa1a66e1ad7d3fba251858","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"91e2917906e893c4adf51d8e3ad38930cff19ee5cb3009d04823d9b45fe41908","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-fwx5gk\" with version 4.21.32 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a50280\u003e: \n failed to create HCP cluster cluster-zstream-4-21-fwx5gk: failed waiting for cluster=\"cluster-zstream-4-21-fwx5gk\" in resourcegroup=\"rg-zstream-upgrade-4-21-b5ssflnhqcdd\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b695f784-3091-465d-8026-55bd201f7208\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b695f784-3091-465d-8026-55bd201f7208\",\n \"name\": \"b695f784-3091-465d-8026-55bd201f7208\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.383849125Z\",\n \"endTime\": \"2026-09-17T02:31:34.01474185Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-21-fwx5gk\" with version 4.21.32 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a50280\u003e: \n failed to create HCP cluster cluster-zstream-4-21-fwx5gk: failed waiting for cluster=\"cluster-zstream-4-21-fwx5gk\" in resourcegroup=\"rg-zstream-upgrade-4-21-b5ssflnhqcdd\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b695f784-3091-465d-8026-55bd201f7208\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b695f784-3091-465d-8026-55bd201f7208\",\n \"name\": \"b695f784-3091-465d-8026-55bd201f7208\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.383849125Z\",\n \"endTime\": \"2026-09-17T02:31:34.01474185Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"0d8a8a69e29ef54537dd3d50194b9a75e5fc91e2f578225b2dba9202edec919f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"0ea6e92c609e741767496ed0bca205dd97df87334b05401151ca3d9e4297e4cc","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0aba0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ms9kkl5qh4rm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n \"name\": \"7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:38.567378599Z\",\n \"endTime\": \"2026-09-17T02:31:43.962353158Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d0aba0\u003e: \n failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ms9kkl5qh4rm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n \"name\": \"7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:38.567378599Z\",\n \"endTime\": \"2026-09-17T02:31:43.962353158Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"11e1601d43590242bf2cca42124831dbb82231e774d93d0fb74103dfcccc9278","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"f188a0c69f93a586d8064b19fecfad91c62cb72340482fa8197c45527b731e94","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a10900\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kf4qgm4z4dts\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/303929de-f9bf-4b87-903f-167135439e71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/303929de-f9bf-4b87-903f-167135439e71\",\n \"name\": \"303929de-f9bf-4b87-903f-167135439e71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:35.753852827Z\",\n \"endTime\": \"2026-09-17T02:31:43.996702977Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:102]: failed to create HCP cluster np-autoscale-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a10900\u003e: \n failed to create HCP cluster np-autoscale-cluster: failed waiting for cluster=\"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kf4qgm4z4dts\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/303929de-f9bf-4b87-903f-167135439e71\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/303929de-f9bf-4b87-903f-167135439e71\",\n \"name\": \"303929de-f9bf-4b87-903f-167135439e71\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:35.753852827Z\",\n \"endTime\": \"2026-09-17T02:31:43.996702977Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"21ad4b8b6b06dc8c2a21b7144e9db9e9534adbcac999e7dacb5d04679273e5c8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"db18d58c7751ab0b22cdad498cec1c389a7f4bcbaed367974d93a33204cc2434","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-j52qlt with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d16cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-j52qlt: failed waiting for cluster=\"np-version-upgrade-cluster-j52qlt\" in resourcegroup=\"rg-np-version-upgrade-j52qlt-5fp9k8q5vgqd\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9371c790-9005-48c5-ab72-5a96cc6f4ca1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9371c790-9005-48c5-ab72-5a96cc6f4ca1\",\n \"name\": \"9371c790-9005-48c5-ab72-5a96cc6f4ca1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:44.264326971Z\",\n \"endTime\": \"2026-09-17T02:31:54.008286613Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-j52qlt with version 4.20.38\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000d16cc0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-j52qlt: failed waiting for cluster=\"np-version-upgrade-cluster-j52qlt\" in resourcegroup=\"rg-np-version-upgrade-j52qlt-5fp9k8q5vgqd\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9371c790-9005-48c5-ab72-5a96cc6f4ca1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9371c790-9005-48c5-ab72-5a96cc6f4ca1\",\n \"name\": \"9371c790-9005-48c5-ab72-5a96cc6f4ca1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:44.264326971Z\",\n \"endTime\": \"2026-09-17T02:31:54.008286613Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"288c4ba1a4482cde8b1bc8ee0c24066200f248b8023a8f14eedf6c419549f44c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"6d8549ce7ce3d738e38f7dd73b06fae91adfeb0e1534bba97833a20ae84d2f8a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-6hvxbh\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016d4240\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-6hvxbh: failed waiting for cluster=\"cp-ystream-upgrade-4-22-6hvxbh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-xh4hgks62ljw\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/531e1a1c-60be-4fce-b365-0f884cff7c52\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/531e1a1c-60be-4fce-b365-0f884cff7c52\",\n \"name\": \"531e1a1c-60be-4fce-b365-0f884cff7c52\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:46.217261407Z\",\n \"endTime\": \"2026-09-17T02:31:53.884522774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-4-22-6hvxbh\" at install version 4.21\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0016d4240\u003e: \n failed to create HCP cluster cp-ystream-upgrade-4-22-6hvxbh: failed waiting for cluster=\"cp-ystream-upgrade-4-22-6hvxbh\" in resourcegroup=\"rg-cp-ystream-upgrade-4-22-xh4hgks62ljw\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/531e1a1c-60be-4fce-b365-0f884cff7c52\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/531e1a1c-60be-4fce-b365-0f884cff7c52\",\n \"name\": \"531e1a1c-60be-4fce-b365-0f884cff7c52\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:46.217261407Z\",\n \"endTime\": \"2026-09-17T02:31:53.884522774Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2b3806e5a80195cb3d7a4e1fde554ef7c11270ded4b127a724b924da31f2838f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"a4ab9812dd60d9325a7127735e40ad640073cecf361f77bd3c75a60c7b17bd98","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0014fc8a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zvg8dvqqf6zz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zvg8dvqqf6zz, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc0014fc8a0\u003e: \n failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zvg8dvqqf6zz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zvg8dvqqf6zz, error: context deadline exceeded\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded"}},{"row_id":"2c04aa75abbcd7dd776219e1ef125cc0bffcad1b2b3e4a4f8230781cf97b64f1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"626800ce28af5178db2305452aa40119998bcbdd63d53a7168853b1c7e730bd3","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-h6jm2t at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001542b80\u003e: \n failed to create HCP cluster np-skip-minor-h6jm2t: failed waiting for cluster=\"np-skip-minor-h6jm2t\" in resourcegroup=\"rg-np-skip-minor-h6jm2t-w4hwhznsjjmk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b45e983d-6f18-4b9a-9811-927f20f1037e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b45e983d-6f18-4b9a-9811-927f20f1037e\",\n \"name\": \"b45e983d-6f18-4b9a-9811-927f20f1037e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:44.420054544Z\",\n \"endTime\": \"2026-09-17T02:31:53.95763018Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:441]: failed to create HCP cluster np-skip-minor-h6jm2t at version 4.22.14\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001542b80\u003e: \n failed to create HCP cluster np-skip-minor-h6jm2t: failed waiting for cluster=\"np-skip-minor-h6jm2t\" in resourcegroup=\"rg-np-skip-minor-h6jm2t-w4hwhznsjjmk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b45e983d-6f18-4b9a-9811-927f20f1037e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b45e983d-6f18-4b9a-9811-927f20f1037e\",\n \"name\": \"b45e983d-6f18-4b9a-9811-927f20f1037e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:44.420054544Z\",\n \"endTime\": \"2026-09-17T02:31:53.95763018Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"2f7d7310d1fd3148cc3596331f4a780e7af6c1abad6e041fd4edd37f1a3fd9f8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"2f9108fe18776541cbc85ef378b8f266052150ee76b5c630519c31e933ddb5b9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-6ztsf49s4tcb/cluster-candidate-4-22-6xdcfg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0c3e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-6xdcfg: failed waiting for cluster=\"cluster-candidate-4-22-6xdcfg\" in resourcegroup=\"rg-candidate-4-22-6ztsf49s4tcb\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a77c9b3c-5b4c-4e42-aab0-8f5e7e0965c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a77c9b3c-5b4c-4e42-aab0-8f5e7e0965c5\",\n \"name\": \"a77c9b3c-5b4c-4e42-aab0-8f5e7e0965c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:48.570738366Z\",\n \"endTime\": \"2026-09-17T02:31:53.987437394Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-22-6ztsf49s4tcb/cluster-candidate-4-22-6xdcfg should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c0c3e0\u003e: \n failed to create HCP cluster cluster-candidate-4-22-6xdcfg: failed waiting for cluster=\"cluster-candidate-4-22-6xdcfg\" in resourcegroup=\"rg-candidate-4-22-6ztsf49s4tcb\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a77c9b3c-5b4c-4e42-aab0-8f5e7e0965c5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/a77c9b3c-5b4c-4e42-aab0-8f5e7e0965c5\",\n \"name\": \"a77c9b3c-5b4c-4e42-aab0-8f5e7e0965c5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:48.570738366Z\",\n \"endTime\": \"2026-09-17T02:31:53.987437394Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"390f3c8e625f557e629c66a73bf12f00e8c587ca576c7b939bafd17cf5b82ab6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"39e9738b3570d1ad49126bbe45de1c14c1222d36b148883b549374de80d01c71","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7e9e0\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-225czhknm7jg\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb86c217-7005-44ed-b917-c04114674f41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb86c217-7005-44ed-b917-c04114674f41\",\n \"name\": \"cb86c217-7005-44ed-b917-c04114674f41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.419178407Z\",\n \"endTime\": \"2026-09-17T02:31:33.944859309Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f7e9e0\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-225czhknm7jg\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb86c217-7005-44ed-b917-c04114674f41\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cb86c217-7005-44ed-b917-c04114674f41\",\n \"name\": \"cb86c217-7005-44ed-b917-c04114674f41\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.419178407Z\",\n \"endTime\": \"2026-09-17T02:31:33.944859309Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"3b20b73d50317f5a8ce73996df240811815389fd74ecdc6edaf33cb65fdcc9de","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"24e450fae1afaf3835ed114031825dba965e2638d9e2a4b78361d363bc7d139c","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4a660\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-96b6xnkpp85l\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bd48285-8d33-4820-9697-a776911193ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bd48285-8d33-4820-9697-a776911193ac\",\n \"name\": \"3bd48285-8d33-4820-9697-a776911193ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.231273638Z\",\n \"endTime\": \"2026-09-17T02:31:34.016609783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e4a660\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-96b6xnkpp85l\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bd48285-8d33-4820-9697-a776911193ac\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bd48285-8d33-4820-9697-a776911193ac\",\n \"name\": \"3bd48285-8d33-4820-9697-a776911193ac\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.231273638Z\",\n \"endTime\": \"2026-09-17T02:31:34.016609783Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"42045b0a2e6f889470ef368b2e3711dcf52527987bb62ea6218e05bb048f19e6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"aa2abdeccf981853abdf8d84e7171612f5443e4af2a9fb66bb42c41ddd759ab7","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc360\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9z5s8nc9xqgx\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b214ddd4-ba76-4a5e-a613-4c36e58e84d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b214ddd4-ba76-4a5e-a613-4c36e58e84d0\",\n \"name\": \"b214ddd4-ba76-4a5e-a613-4c36e58e84d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:41.948718676Z\",\n \"endTime\": \"2026-09-17T02:31:43.891046142Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:89]: failed to create HCP cluster for KMS key rotation test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fbc360\u003e: \n failed to create HCP cluster kms-key-rotate-422: failed waiting for cluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-9z5s8nc9xqgx\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b214ddd4-ba76-4a5e-a613-4c36e58e84d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/b214ddd4-ba76-4a5e-a613-4c36e58e84d0\",\n \"name\": \"b214ddd4-ba76-4a5e-a613-4c36e58e84d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:41.948718676Z\",\n \"endTime\": \"2026-09-17T02:31:43.891046142Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"43c11a38480495e6f79ada754a5417113b29bd0330d4dd4f0249a4bd2a15605b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"alert","signature_id":"4d675a2aa4e866465ba35a460544e188e20719f62123eb05ebda79f62db3498b","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T02:39:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T02:40:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T02:46:35Z\n Ended: 2026-09-17T02:54:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T02:46:35Z\n Ended: 2026-09-17T02:59:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n Description: Rollout of deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired"}},{"row_id":"5247511af8453ab04fe5089ac8fc5cb307bd11e8a00b0cbc2deaf08ea154f107","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"2e37c10dee44a98ad4c5d8e21f945e822bc6d1d157c0aec08182d653353bf301","test_name":"SRE should be able to retrieve serial console logs for a VM","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048320\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-s6bm4zgqdnrw\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cbb25a4-33cb-4061-b0a5-4f75945aa4d9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cbb25a4-33cb-4061-b0a5-4f75945aa4d9\",\n \"name\": \"3cbb25a4-33cb-4061-b0a5-4f75945aa4d9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:38.356951393Z\",\n \"endTime\": \"2026-09-17T02:31:43.908352124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001048320\u003e: \n failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-s6bm4zgqdnrw\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cbb25a4-33cb-4061-b0a5-4f75945aa4d9\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3cbb25a4-33cb-4061-b0a5-4f75945aa4d9\",\n \"name\": \"3cbb25a4-33cb-4061-b0a5-4f75945aa4d9\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:38.356951393Z\",\n \"endTime\": \"2026-09-17T02:31:43.908352124Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"54069facbba00eb377b521ba5a0f483f972d7206ffe92065acb7be02d2f0f432","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"9020e6f833dc38c47ccef3d4182afa72cebc722f85d46ac12ebd56299acce896","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90a40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8x87dzwxvb96\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n \"name\": \"3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:04:32.691734276Z\",\n \"endTime\": \"2026-09-17T02:33:33.968010236Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f90a40\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8x87dzwxvb96\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n \"name\": \"3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:04:32.691734276Z\",\n \"endTime\": \"2026-09-17T02:33:33.968010236Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"55fd677673f58af31e171e5c6256b40029470706b8bd4c699bae87207cd06e46","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"2161ec92ec6ee1b1f4126d8043c75c164aa59ccae1d75ae8fdecc279478acefb","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b92880\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-z57jsd9qjtzf\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84933e1d-1d50-47a1-a019-2e5e7e28d1ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84933e1d-1d50-47a1-a019-2e5e7e28d1ba\",\n \"name\": \"84933e1d-1d50-47a1-a019-2e5e7e28d1ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:44.303522195Z\",\n \"endTime\": \"2026-09-17T02:31:53.875776363Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b92880\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-z57jsd9qjtzf\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84933e1d-1d50-47a1-a019-2e5e7e28d1ba\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/84933e1d-1d50-47a1-a019-2e5e7e28d1ba\",\n \"name\": \"84933e1d-1d50-47a1-a019-2e5e7e28d1ba\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:44.303522195Z\",\n \"endTime\": \"2026-09-17T02:31:53.875776363Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"561cda425d85e0b51b856b6d06d2d539f247d3c8144ffe32b8bb3ef092055c92","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"7abcaf7cf828a86d797ada0bafb042a8494db58e91375d1805c53de72f0578cb","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012be260\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-f2s6nsbcb4km\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96826b81-1ed0-4935-b83d-e068c4aaec7e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96826b81-1ed0-4935-b83d-e068c4aaec7e\",\n \"name\": \"96826b81-1ed0-4935-b83d-e068c4aaec7e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:38.422604462Z\",\n \"endTime\": \"2026-09-17T02:31:44.011868414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0012be260\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-f2s6nsbcb4km\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96826b81-1ed0-4935-b83d-e068c4aaec7e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/96826b81-1ed0-4935-b83d-e068c4aaec7e\",\n \"name\": \"96826b81-1ed0-4935-b83d-e068c4aaec7e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:38.422604462Z\",\n \"endTime\": \"2026-09-17T02:31:44.011868414Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6458cf80e9e2aa9d8c7d34b8f3682c9ffb731d43a307c9f508ddc6989532b114","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"alert","signature_id":"9a8fc1c68aeeef65c0fa2d8774b771f9f44e0b8d2622880bad833fdcea513992","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T02:04:33Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-6f59c7c94f-hsk8t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-6f59c7c94f-hsk8t has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired"}},{"row_id":"66b8312846550257bbbf31b05e20d0d110393f0335018ca6bcc6573066851cb9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"593e3571cb87eb1c990363d1e184cfd091bfa920f1d8233ab9b8bffafc8753f4","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bee140\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n2zqcqmlz6pc\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2786d7d-1f11-4c6b-b3c4-ecb9d5711378\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2786d7d-1f11-4c6b-b3c4-ecb9d5711378\",\n \"name\": \"c2786d7d-1f11-4c6b-b3c4-ecb9d5711378\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:27.795928391Z\",\n \"endTime\": \"2026-09-17T02:31:33.962956757Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:88]: failed to create HCP cluster \"arm64-vm-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000bee140\u003e: \n failed to create HCP cluster arm64-vm-hcp-cluster: failed waiting for cluster=\"arm64-vm-hcp-cluster\" in resourcegroup=\"arm64-vm-cluster-n2zqcqmlz6pc\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2786d7d-1f11-4c6b-b3c4-ecb9d5711378\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2786d7d-1f11-4c6b-b3c4-ecb9d5711378\",\n \"name\": \"c2786d7d-1f11-4c6b-b3c4-ecb9d5711378\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:27.795928391Z\",\n \"endTime\": \"2026-09-17T02:31:33.962956757Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"66ceeaaab1ad90937b75b08fbda29997e9054127da73447ce2e82e19939b20b9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed","failure_pattern_match":{"status":"clustered","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed"}},{"row_id":"6e9fd00cffda08761b34ee2eff8f0189c92a3a85e957fc3bc05c88b34c190793","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"451e15ada92473b46c37ac7d85751c6c542d9156788736edb08510846c807aa5","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d8f20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-5jtz9dz48t8w\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a755a8a-369f-4b08-a5a5-89db9b41a341\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a755a8a-369f-4b08-a5a5-89db9b41a341\",\n \"name\": \"4a755a8a-369f-4b08-a5a5-89db9b41a341\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.405905627Z\",\n \"endTime\": \"2026-09-17T02:31:34.009715425Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006d8f20\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-5jtz9dz48t8w\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a755a8a-369f-4b08-a5a5-89db9b41a341\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4a755a8a-369f-4b08-a5a5-89db9b41a341\",\n \"name\": \"4a755a8a-369f-4b08-a5a5-89db9b41a341\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.405905627Z\",\n \"endTime\": \"2026-09-17T02:31:34.009715425Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"6fe3f443d4ea9794c07fe0b776a1e16cbfe82587fe0b096a06cebeb34e32a66f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"alert","signature_id":"48e28661b4bfc093fce3e4eccd4ac9c8f3996f35cf8b87e4d8ccb1de279e00a8","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 16 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T02:29:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T02:30:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-17T02:35:35Z\n Ended: 2026-09-17T02:42:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfodlh52rjsn8v0i6tqm6t35c2l0p0\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfodlh52rjsn8v0i6tqm6t35c2l0p0/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-17T02:35:35Z\n Ended: 2026-09-17T02:42:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfoc87t8ajfhu9n4pqjtbphn862gj1\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfoc87t8ajfhu9n4pqjtbphn862gj1/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-17T02:35:35Z\n Ended: 2026-09-17T02:42:39Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfoc8cioja4t99v8qf8fgteg8hp8cq\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfoc8cioja4t99v8qf8fgteg8hp8cq/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-17T02:35:35Z\n Ended: 2026-09-17T02:42:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfofif68524qhonc25k6st2irl6nvr\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfofif68524qhonc25k6st2irl6nvr/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-17T02:36:36Z\n Ended: 2026-09-17T02:42:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfodboatgvtrijvb9itq8g6nv3a0lj\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfodboatgvtrijvb9itq8g6nv3a0lj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-17T02:36:36Z\n Ended: 2026-09-17T02:42:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfod4unnpb3fidghon0kvgohtrm0gi\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfod4unnpb3fidghon0kvgohtrm0gi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-17T02:36:36Z\n Ended: 2026-09-17T02:42:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfod7a9s6uqchgri0b4afl9dc7n19f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfod7a9s6uqchgri0b4afl9dc7n19f/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-17T02:36:36Z\n Ended: 2026-09-17T02:42:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfoda33s4gbilpj50f3r2392mijctf\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfoda33s4gbilpj50f3r2392mijctf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-17T02:36:36Z\n Ended: 2026-09-17T02:43:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfodmtcgprbt25mkk7n1f31gp57v5p\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfodmtcgprbt25mkk7n1f31gp57v5p/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-17T02:36:36Z\n Ended: 2026-09-17T02:59:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-17T02:36:36Z\n Ended: 2026-09-17T02:42:39Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfofsi9cr1hrnf33j5m2j2v0anob00\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfofsi9cr1hrnf33j5m2j2v0anob00/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-17T02:36:36Z\n Ended: 2026-09-17T02:54:35Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-17T02:36:36Z\n Ended: 2026-09-17T02:42:38Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfof0kh0mr490ilc2tusdfihbkudsb\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfof0kh0mr490ilc2tusdfihbkudsb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-17T02:37:34Z\n Ended: 2026-09-17T02:43:34Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfohaljjdbuho1cfahj1o9d3dj04u4\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2ssfohaljjdbuho1cfahj1o9d3dj04u4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired"}},{"row_id":"7d310bcbee54a598cb60cf66bc6553e2fba267f4a95313527b7976351bb7ec43","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"02219053511d8a21950017349ee31494aa267bbe162808b309c406afe57bd88e","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110a440\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qr9h99g92762\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n \"name\": \"c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.114427313Z\",\n \"endTime\": \"2026-09-17T02:31:33.96482384Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00110a440\u003e: \n failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qr9h99g92762\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n \"name\": \"c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.114427313Z\",\n \"endTime\": \"2026-09-17T02:31:33.96482384Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8ae216ba8abc294c1ac91b58b39248d2ccbe6dc7ff1fff81baf62592f338870c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"417a674aaec0692852dbca6d977cc89fe23494c77a5a0606b6bafecdf7722e62","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-j9twqf with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e3a4a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-j9twqf: failed waiting for cluster=\"np-version-upgrade-cluster-j9twqf\" in resourcegroup=\"rg-np-version-upgrade-j9twqf-b7p8nsv9crgv\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d04b62a-ee8e-4fce-b798-1257698c18c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d04b62a-ee8e-4fce-b798-1257698c18c7\",\n \"name\": \"7d04b62a-ee8e-4fce-b798-1257698c18c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:44.709948422Z\",\n \"endTime\": \"2026-09-17T02:31:54.011799797Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-j9twqf with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000e3a4a0\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-j9twqf: failed waiting for cluster=\"np-version-upgrade-cluster-j9twqf\" in resourcegroup=\"rg-np-version-upgrade-j9twqf-b7p8nsv9crgv\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d04b62a-ee8e-4fce-b798-1257698c18c7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d04b62a-ee8e-4fce-b798-1257698c18c7\",\n \"name\": \"7d04b62a-ee8e-4fce-b798-1257698c18c7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:44.709948422Z\",\n \"endTime\": \"2026-09-17T02:31:54.011799797Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"8fa2cd7b363376a12838bcdab5ac99e16d80d547377bbd2a07980f9b4bb34233","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"d166da666940169c48a559831401ddc71f07bc674b8bac26199e6f32e66bd324","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kjzx5h with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf0980\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-kjzx5h: failed waiting for cluster=\"np-version-upgrade-cluster-kjzx5h\" in resourcegroup=\"rg-np-version-upgrade-kjzx5h-cfkg25kg5pff\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/147a24b3-2ffd-49e8-a97a-6e6fca89fafe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/147a24b3-2ffd-49e8-a97a-6e6fca89fafe\",\n \"name\": \"147a24b3-2ffd-49e8-a97a-6e6fca89fafe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:24.251677536Z\",\n \"endTime\": \"2026-09-17T02:31:24.046042722Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-kjzx5h with version 4.21.33\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cf0980\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-kjzx5h: failed waiting for cluster=\"np-version-upgrade-cluster-kjzx5h\" in resourcegroup=\"rg-np-version-upgrade-kjzx5h-cfkg25kg5pff\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/147a24b3-2ffd-49e8-a97a-6e6fca89fafe\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/147a24b3-2ffd-49e8-a97a-6e6fca89fafe\",\n \"name\": \"147a24b3-2ffd-49e8-a97a-6e6fca89fafe\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:24.251677536Z\",\n \"endTime\": \"2026-09-17T02:31:24.046042722Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"a33a52a57c76791841a11f0e4eb563dac9ac16dbec2f04c33231985f72e2d4da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment"}},{"row_id":"a94bf06c540e1aa15a2de626c9ec761a86fa3f7d8ce28eab15b65b24ebce2d6c","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"e19ccc38c497a63fa54dc54b1393fcc47c00e94ab08b2c73966853333434a251","test_name":"Customer should not be able to perform various invalid operations on cluster resources","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001152b40\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-4tlzr7kc2655\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04eb485b-d159-4a7e-a6b0-ef38fce18568\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04eb485b-d159-4a7e-a6b0-ef38fce18568\",\n \"name\": \"04eb485b-d159-4a7e-a6b0-ef38fce18568\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:26:41.290488511Z\",\n \"endTime\": \"2026-09-17T02:55:43.925125024Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:110]: failed to create HCP cluster negative-tests-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001152b40\u003e: \n failed to create HCP cluster negative-tests-cluster: failed waiting for cluster=\"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-4tlzr7kc2655\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04eb485b-d159-4a7e-a6b0-ef38fce18568\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/04eb485b-d159-4a7e-a6b0-ef38fce18568\",\n \"name\": \"04eb485b-d159-4a7e-a6b0-ef38fce18568\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:26:41.290488511Z\",\n \"endTime\": \"2026-09-17T02:55:43.925125024Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ba44395502ff82a7e6545359bd7588625b3c281f6ea3da9ce60f80b39310b591","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"139bcc210692fb4c4608f88c1e62ffbae5cc26a9ea38e8a550f86dc64be5bb37","test_name":"SRE should be able to log into a cluster via a breakglass session","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cdc740\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9lhfrbsqkhjk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n \"name\": \"39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:27.808727421Z\",\n \"endTime\": \"2026-09-17T02:31:33.989605445Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cdc740\u003e: \n failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9lhfrbsqkhjk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n \"name\": \"39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:27.808727421Z\",\n \"endTime\": \"2026-09-17T02:31:33.989605445Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c1cf3afb625b9b4da02a78d66eecc551e595cbaae3d959e2101e70e264b39e65","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"c8695c56c9968b93a6495331c031bfc4f9defddf58bdeb3d72666b657cc7d8c5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-zmcbpxcpx6ws/cluster-candidate-4-21-cqz9fw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010085c0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-cqz9fw: failed waiting for cluster=\"cluster-candidate-4-21-cqz9fw\" in resourcegroup=\"rg-candidate-4-21-zmcbpxcpx6ws\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e9958ce-766f-4036-b7e8-0b661be55619\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e9958ce-766f-4036-b7e8-0b661be55619\",\n \"name\": \"9e9958ce-766f-4036-b7e8-0b661be55619\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:49.673438329Z\",\n \"endTime\": \"2026-09-17T02:31:53.992657772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-zmcbpxcpx6ws/cluster-candidate-4-21-cqz9fw should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010085c0\u003e: \n failed to create HCP cluster cluster-candidate-4-21-cqz9fw: failed waiting for cluster=\"cluster-candidate-4-21-cqz9fw\" in resourcegroup=\"rg-candidate-4-21-zmcbpxcpx6ws\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e9958ce-766f-4036-b7e8-0b661be55619\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9e9958ce-766f-4036-b7e8-0b661be55619\",\n \"name\": \"9e9958ce-766f-4036-b7e8-0b661be55619\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:49.673438329Z\",\n \"endTime\": \"2026-09-17T02:31:53.992657772Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"c394e19efaa9e8d343b61505f2880e7192c9610b454e08f19137cc03ceb61142","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"9e87469877ef0917409c14d4d33b60f1b2ccdcdd81485077bc197c0d9d5a5e97","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001330460\u003e: \n failed to create HCP cluster np-dg-minor-ppm5ds: failed waiting for cluster=\"np-dg-minor-ppm5ds\" in resourcegroup=\"rg-np-dg-minor-ppm5ds-qsf8p9hp8hdm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/53af4dc3-7144-40af-8942-346948cc38f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/53af4dc3-7144-40af-8942-346948cc38f8\",\n \"name\": \"53af4dc3-7144-40af-8942-346948cc38f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:24.259334609Z\",\n \"endTime\": \"2026-09-17T02:31:24.01732328Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:720]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001330460\u003e: \n failed to create HCP cluster np-dg-minor-ppm5ds: failed waiting for cluster=\"np-dg-minor-ppm5ds\" in resourcegroup=\"rg-np-dg-minor-ppm5ds-qsf8p9hp8hdm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/53af4dc3-7144-40af-8942-346948cc38f8\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/53af4dc3-7144-40af-8942-346948cc38f8\",\n \"name\": \"53af4dc3-7144-40af-8942-346948cc38f8\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:24.259334609Z\",\n \"endTime\": \"2026-09-17T02:31:24.01732328Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e7e6aa09265fb8f2ca524df0a52cfec2bd99ee2e5a8f4af5a0b35ac8948d5447","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"284fd36dc5ef5d74feffb6a93226bcfff835efdad8f1f85bdf032a8053cc1b6a","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a7420\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jt498f98p44p\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d24c8171-8d46-426a-8e8c-f4e850903936\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d24c8171-8d46-426a-8e8c-f4e850903936\",\n \"name\": \"d24c8171-8d46-426a-8e8c-f4e850903936\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:37.263414559Z\",\n \"endTime\": \"2026-09-17T02:31:43.998529874Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:94]: failed to create HCP cluster for external auth test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0008a7420\u003e: \n failed to create HCP cluster ea-cluster: failed waiting for cluster=\"ea-cluster\" in resourcegroup=\"external-auth-cluster-jt498f98p44p\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d24c8171-8d46-426a-8e8c-f4e850903936\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d24c8171-8d46-426a-8e8c-f4e850903936\",\n \"name\": \"d24c8171-8d46-426a-8e8c-f4e850903936\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:37.263414559Z\",\n \"endTime\": \"2026-09-17T02:31:43.998529874Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"f0b051f0de0a2d6003170fdda4270b708b493b2f34e896b7fb564562ff7948fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","failed_at":"e2e","signature_id":"9e699244c91afd46d8472b6ccd3aa4e2a0f99db87af440b007dcf8abcbb3ce6d","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-v7k2jv\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001184300\u003e: \n failed to create HCP cluster cluster-zstream-4-22-v7k2jv: failed waiting for cluster=\"cluster-zstream-4-22-v7k2jv\" in resourcegroup=\"rg-zstream-upgrade-4-22-w2swgqxn7qgg\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3738bf13-d603-464c-9ea8-a79b06759e18\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3738bf13-d603-464c-9ea8-a79b06759e18\",\n \"name\": \"3738bf13-d603-464c-9ea8-a79b06759e18\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.426824456Z\",\n \"endTime\": \"2026-09-17T02:31:33.942611429Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-22-v7k2jv\" with version 4.22.13 on candidate channel\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001184300\u003e: \n failed to create HCP cluster cluster-zstream-4-22-v7k2jv: failed waiting for cluster=\"cluster-zstream-4-22-v7k2jv\" in resourcegroup=\"rg-zstream-upgrade-4-22-w2swgqxn7qgg\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3738bf13-d603-464c-9ea8-a79b06759e18\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3738bf13-d603-464c-9ea8-a79b06759e18\",\n \"name\": \"3738bf13-d603-464c-9ea8-a79b06759e18\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:02:28.426824456Z\",\n \"endTime\": \"2026-09-17T02:31:33.942611429Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":34,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8"],"matched_occurrences":34,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6974,"pr_state":"closed","pr_sha":"ad18c381b0fe07c50afb22f07e5b849b9d525738","final_merged_sha":"fdb1b526d51e98ec4ae2b5a2fc9fae30dbe31d6d","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-09-17T00:56:45Z","started_at":"2026-09-17T00:56:45Z","completed_at":"2026-09-17T03:54:37Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:04.070198597Z","timing_metadata_checked_at":"2026-09-18T15:41:04.070198597Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:04.073360508Z","region_metadata_checked_at":"2026-09-18T15:41:04.073360508Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"5c6c5cf7832bb10a5c7abd71d1586e41b5eef48c293ac71a175a26949b6f33f4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","failed_at":"e2e","signature_id":"3287e14a7b0a164b12f072d969ca8183f1647cf1f1f00a66de841c81d64ef6e4","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a844a0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-mdr5vl8z5h7q\" to finish creating: GET https://rp.j6437632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1372c7a4-2df6-4180-8631-c7a6144c4c99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1372c7a4-2df6-4180-8631-c7a6144c4c99\",\n \"name\": \"1372c7a4-2df6-4180-8631-c7a6144c4c99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:44:45.592258788Z\",\n \"endTime\": \"2026-09-17T03:13:52.849263315Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m6s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a844a0\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-mdr5vl8z5h7q\" to finish creating: GET https://rp.j6437632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1372c7a4-2df6-4180-8631-c7a6144c4c99\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1372c7a4-2df6-4180-8631-c7a6144c4c99\",\n \"name\": \"1372c7a4-2df6-4180-8631-c7a6144c4c99\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:44:45.592258788Z\",\n \"endTime\": \"2026-09-17T03:13:52.849263315Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 21m6s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"708e81b82a424169248899ae8d08c68fdf1eeb45f661db99c1208fa7d45ce3ee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","failed_at":"e2e","signature_id":"a5eed4f538e16bf946e6c1d7bf29b73fc650cf9bfa21cbd95eec591f73113f97","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vl7c2rjpvjq7/cluster-candidate-4-21-p7js6n should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b60e20\u003e: \n failed to create HCP cluster cluster-candidate-4-21-p7js6n: failed waiting for cluster=\"cluster-candidate-4-21-p7js6n\" in resourcegroup=\"rg-candidate-4-21-vl7c2rjpvjq7\" to finish creating: GET https://rp.j6437632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67d55c5a-1df5-45a1-9d8a-80663ee1244f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67d55c5a-1df5-45a1-9d8a-80663ee1244f\",\n \"name\": \"67d55c5a-1df5-45a1-9d8a-80663ee1244f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:44:45.387823311Z\",\n \"endTime\": \"2026-09-17T03:13:52.879414342Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m50s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:122]: HCP cluster rg-candidate-4-21-vl7c2rjpvjq7/cluster-candidate-4-21-p7js6n should provision\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000b60e20\u003e: \n failed to create HCP cluster cluster-candidate-4-21-p7js6n: failed waiting for cluster=\"cluster-candidate-4-21-p7js6n\" in resourcegroup=\"rg-candidate-4-21-vl7c2rjpvjq7\" to finish creating: GET https://rp.j6437632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67d55c5a-1df5-45a1-9d8a-80663ee1244f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67d55c5a-1df5-45a1-9d8a-80663ee1244f\",\n \"name\": \"67d55c5a-1df5-45a1-9d8a-80663ee1244f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:44:45.387823311Z\",\n \"endTime\": \"2026-09-17T03:13:52.879414342Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m50s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"866fe6d8b9dd50d63a8087839f052541e2cfff26e6dba7671f8c0adbe2f4760b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","failed_at":"alert","signature_id":"3ddbbbbfcbe35abb46fc03a012581caf1092f9cbcef18f864c7ff0cb4097904d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T02:59:00Z\n Ended: 2026-09-17T03:27:06Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6437632-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssgc7v2iqv4rfbckr31morckojl5215-private-ingress\", pod=\"router-667f75c547-v8wkq\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssgc7v2iqv4rfbckr31morckojl5215-private-ingress/router-667f75c547-v8wkq has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T03:01:04Z\n Ended: 2026-09-17T03:24:03Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j6437632-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssgc7n9j9qjhfviqdpt5g6ua6btqmcn-h8g3i4v4t3m5a2s\", pod=\"router-76b66bfcb5-vt5tw\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssgc7n9j9qjhfviqdpt5g6ua6btqmcn-h8g3i4v4t3m5a2s/router-76b66bfcb5-vt5tw has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"a53ba2040b32d72354d2b0c6f2e788e72077c93314d99f5e6ea264061aac25d0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","failed_at":"alert","signature_id":"f729f652c9b63e98893c72b91f3a1326eda28d14e829f566594109b7326b49cb","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 15 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-17T03:16:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-797c67bcdc-pfftn\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-797c67bcdc-pfftn (namespace clusters-service) on cluster ci01-j6437632-svc has averaged 242.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T03:16:00Z\n Ended: 2026-09-17T03:36:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-66f56f4b8b-mmnm8\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-66f56f4b8b-mmnm8 (namespace mgmt-agent) on cluster ci01-j6437632-mgmt-1 has averaged 172.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Fired (not resolved)\n Started: 2026-09-17T03:16:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-797c67bcdc-m7rk4\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-797c67bcdc-m7rk4 (namespace clusters-service) on cluster ci01-j6437632-svc has averaged 751.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T03:16:00Z\n Ended: 2026-09-17T03:43:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-gnk8z\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-gnk8z (namespace arobit) on cluster ci01-j6437632-svc has averaged 186.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Fired (not resolved)\n Started: 2026-09-17T03:16:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-797c67bcdc-n48gs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-797c67bcdc-n48gs (namespace clusters-service) on cluster ci01-j6437632-svc has averaged 286.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Resolved\n Started: 2026-09-17T03:16:00Z\n Ended: 2026-09-17T03:32:57Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5snrs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5snrs (namespace arobit) on cluster ci01-j6437632-mgmt-2 has averaged 140.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Fired (not resolved)\n Started: 2026-09-17T03:16:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-5f5946cd66-9mq4p\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-5f5946cd66-9mq4p (namespace aro-hcp) on cluster ci01-j6437632-svc has averaged 424.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T03:16:00Z\n Ended: 2026-09-17T03:32:57Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-55c58\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-55c58 (namespace arobit) on cluster ci01-j6437632-mgmt-1 has averaged 141.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Resolved\n Started: 2026-09-17T03:16:56Z\n Ended: 2026-09-17T03:35:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-66f56f4b8b-9fks7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-66f56f4b8b-9fks7 (namespace mgmt-agent) on cluster ci01-j6437632-mgmt-2 has averaged 164.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T03:17:56Z\n Ended: 2026-09-17T03:36:59Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-rdp9x\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-rdp9x (namespace arobit) on cluster ci01-j6437632-mgmt-1 has averaged 147.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-17T03:18:57Z\n Ended: 2026-09-17T03:40:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-ljkp7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-ljkp7 (namespace arobit) on cluster ci01-j6437632-mgmt-2 has averaged 165.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n State: Resolved\n Started: 2026-09-17T03:22:00Z\n Ended: 2026-09-17T03:29:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fcrcj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fcrcj (namespace arobit) on cluster ci01-j6437632-svc has averaged 128.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 13:\n State: Resolved\n Started: 2026-09-17T03:22:00Z\n Ended: 2026-09-17T03:34:59Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2hwp7\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-2hwp7 (namespace arobit) on cluster ci01-j6437632-mgmt-2 has averaged 129% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 14:\n State: Resolved\n Started: 2026-09-17T03:23:58Z\n Ended: 2026-09-17T03:38:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-62bl5\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-62bl5 (namespace arobit) on cluster ci01-j6437632-mgmt-1 has averaged 142.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 15:\n State: Resolved\n Started: 2026-09-17T03:25:58Z\n Ended: 2026-09-17T03:31:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6437632-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-hbpdt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-hbpdt (namespace arobit) on cluster ci01-j6437632-mgmt-1 has averaged 120.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"c0ec54fa2d5ca1e8902df113d60101c1f62ab2d060b1bc2fc8dd1cb5f66563c9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","failed_at":"e2e","signature_id":"1c3549dc55bc6c387adb5760f50c6e243c8a43f7b39ed0bf467dc5a0511968bd","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c60b00\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qfm9q98mfcpr\" to finish creating: GET https://rp.j6437632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3dcca7a1-32ce-4f14-918d-eb4b07fb190f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3dcca7a1-32ce-4f14-918d-eb4b07fb190f\",\n \"name\": \"3dcca7a1-32ce-4f14-918d-eb4b07fb190f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:44:19.790541307Z\",\n \"endTime\": \"2026-09-17T03:13:21.798847533Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:80]: failed to create HCP cluster np-delete-hcp-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c60b00\u003e: \n failed to create HCP cluster np-delete-hcp-cluster: failed waiting for cluster=\"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qfm9q98mfcpr\" to finish creating: GET https://rp.j6437632.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3dcca7a1-32ce-4f14-918d-eb4b07fb190f\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3dcca7a1-32ce-4f14-918d-eb4b07fb190f\",\n \"name\": \"3dcca7a1-32ce-4f14-918d-eb4b07fb190f\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:44:19.790541307Z\",\n \"endTime\": \"2026-09-17T03:13:21.798847533Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m37s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6894,"pr_state":"closed","pr_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","final_merged_sha":"41a67196c6bda1e3e9054e18743b0c167da289f7","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T00:47:27Z","started_at":"2026-09-17T00:47:27Z","completed_at":"2026-09-17T03:57:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:33.801290995Z","timing_metadata_checked_at":"2026-09-18T15:40:33.801290995Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:40:33.804496606Z","region_metadata_checked_at":"2026-09-18T15:40:33.804496606Z"},"failed_at":["alert","e2e"],"failed_test_count":8,"occurrences":[{"row_id":"59212b1b8ece7bcc057c6e5ebbdd6e94fb9c59bb3b5bfea1070b52ff342a6366","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","failed_at":"e2e","signature_id":"2913423376b3c9240235607ee0d4c675506def5b909b1312a4bdad97516cd24d","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000985200\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4vtg2dzg4tsm\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67e33856-41f6-4643-ae45-a797cf841c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67e33856-41f6-4643-ae45-a797cf841c28\",\n \"name\": \"67e33856-41f6-4643-ae45-a797cf841c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:46:55.291101558Z\",\n \"endTime\": \"2026-09-17T03:15:59.322698766Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000985200\u003e: \n failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-4vtg2dzg4tsm\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67e33856-41f6-4643-ae45-a797cf841c28\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/67e33856-41f6-4643-ae45-a797cf841c28\",\n \"name\": \"67e33856-41f6-4643-ae45-a797cf841c28\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:46:55.291101558Z\",\n \"endTime\": \"2026-09-17T03:15:59.322698766Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 19m15s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"71e3c639834fdf35cc2dffb9145b7d92ff6a30d2de4383e27a904d16f8c2815d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","failed_at":"e2e","signature_id":"428648ff0c0135e57b228a05c39d11591ece01ed750e930d86d83fe847be740f","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010b2f60\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hk9sm6mhjqks\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6a50ceeb-54cc-4ca7-b087-10be0245475a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6a50ceeb-54cc-4ca7-b087-10be0245475a\",\n \"name\": \"6a50ceeb-54cc-4ca7-b087-10be0245475a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:46:17.393539683Z\",\n \"endTime\": \"2026-09-17T03:15:19.381708079Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010b2f60\u003e: \n failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-hk9sm6mhjqks\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6a50ceeb-54cc-4ca7-b087-10be0245475a\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6a50ceeb-54cc-4ca7-b087-10be0245475a\",\n \"name\": \"6a50ceeb-54cc-4ca7-b087-10be0245475a\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:46:17.393539683Z\",\n \"endTime\": \"2026-09-17T03:15:19.381708079Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 18m38s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"772da4513850efd758181d60daf30d9eeac506607e3b8a11081973455f3374ec","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","failed_at":"alert","signature_id":"eeada1c413c3ca5fc2af1c6cbb216ce484bf15370729d3a7b144bcec22517b3c","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 11 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T03:17:31Z\n Ended: 2026-09-17T03:23:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7855548969-ghlbw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7855548969-ghlbw (namespace mgmt-agent) on cluster ci01-j6537856-mgmt-2 has averaged 166.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Resolved\n Started: 2026-09-17T03:17:31Z\n Ended: 2026-09-17T03:36:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-9pnhk\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-9pnhk (namespace arobit) on cluster ci01-j6537856-mgmt-1 has averaged 157.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T03:17:31Z\n Ended: 2026-09-17T03:43:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8vbzs\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-8vbzs (namespace arobit) on cluster ci01-j6537856-mgmt-1 has averaged 182.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T03:17:31Z\n Ended: 2026-09-17T03:30:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-fdc4t\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-fdc4t (namespace arobit) on cluster ci01-j6537856-mgmt-2 has averaged 177.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T03:17:31Z\n Ended: 2026-09-17T03:23:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-7gz8n\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-7gz8n (namespace arobit) on cluster ci01-j6537856-mgmt-2 has averaged 174.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-17T03:17:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-77db6dcf77-r98d2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-77db6dcf77-r98d2 (namespace clusters-service) on cluster ci01-j6537856-svc has averaged 836.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T03:17:31Z\n Ended: 2026-09-17T03:44:28Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-l75wc\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-l75wc (namespace arobit) on cluster ci01-j6537856-svc has averaged 205.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T03:17:31Z\n Ended: 2026-09-17T03:47:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-77db6dcf77-gc7h2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-77db6dcf77-gc7h2 (namespace clusters-service) on cluster ci01-j6537856-svc has averaged 258.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Fired (not resolved)\n Started: 2026-09-17T03:17:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-c4ccdfb64-796h2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-c4ccdfb64-796h2 (namespace aro-hcp) on cluster ci01-j6537856-svc has averaged 408.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T03:17:31Z\n Ended: 2026-09-17T03:46:29Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-77db6dcf77-qq22x\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-77db6dcf77-qq22x (namespace clusters-service) on cluster ci01-j6537856-svc has averaged 242.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-17T03:17:32Z\n Ended: 2026-09-17T03:36:30Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6537856-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7855548969-zkvgl\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-7855548969-zkvgl (namespace mgmt-agent) on cluster ci01-j6537856-mgmt-1 has averaged 144.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"7bac94707f2fa55568fb3060b83941c396d5e4d89beb10f854e3f26dd59cdd0e","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","failed_at":"e2e","signature_id":"721faaab8ec90430d24300487ededdb0a93721318cf4132ab4ace8fde5518506","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001712600\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-2vff89ssj75w\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/433f8dbb-9a26-40e6-b561-8f8f52095538\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/433f8dbb-9a26-40e6-b561-8f8f52095538\",\n \"name\": \"433f8dbb-9a26-40e6-b561-8f8f52095538\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:46:24.868779354Z\",\n \"endTime\": \"2026-09-17T03:15:29.379097107Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m44s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:123]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001712600\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-2vff89ssj75w\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/433f8dbb-9a26-40e6-b561-8f8f52095538\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/433f8dbb-9a26-40e6-b561-8f8f52095538\",\n \"name\": \"433f8dbb-9a26-40e6-b561-8f8f52095538\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:46:24.868779354Z\",\n \"endTime\": \"2026-09-17T03:15:29.379097107Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m44s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"9dfda47afe4502610b4eba10aa8d38cc1c3640c02618d17a9e56c9a307cf1ef3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","failed_at":"e2e","signature_id":"49af2dca8e5069846af20cde9f9397824ce26e8ea2d79c52e7c7b30e8faed262","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cdc120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-p2sqgdjzvkvn\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6185eb10-51bb-4696-a53a-c5eaad9dc31d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6185eb10-51bb-4696-a53a-c5eaad9dc31d\",\n \"name\": \"6185eb10-51bb-4696-a53a-c5eaad9dc31d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:45:22.801463389Z\",\n \"endTime\": \"2026-09-17T03:14:29.359358725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000cdc120\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-p2sqgdjzvkvn\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6185eb10-51bb-4696-a53a-c5eaad9dc31d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/6185eb10-51bb-4696-a53a-c5eaad9dc31d\",\n \"name\": \"6185eb10-51bb-4696-a53a-c5eaad9dc31d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:45:22.801463389Z\",\n \"endTime\": \"2026-09-17T03:14:29.359358725Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m20s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"b8d03fb887045ea5c825b733c9b18eb594bc6c018ffd8dfdcb567a28d5bfb18d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","failed_at":"e2e","signature_id":"d7f93e547e4be2029c411ab0ce7110ad6d4b2c44c4f85c58ccad50e27c80ca41","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a10e80\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8nfj4dg6xpxk\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\",\n \"name\": \"063e2ac2-9588-4f9c-82b9-899a69317856\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:48:07.392021884Z\",\n \"endTime\": \"2026-09-17T03:17:09.326897061Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a10e80\u003e: \n failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8nfj4dg6xpxk\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\",\n \"name\": \"063e2ac2-9588-4f9c-82b9-899a69317856\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:48:07.392021884Z\",\n \"endTime\": \"2026-09-17T03:17:09.326897061Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"ba18440010dc58603161adedd29215fd5271930b55394d1c99b63ef105254e0f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","failed_at":"e2e","signature_id":"0d64bfff34a7c1f61cc366edc7ee533840e5653ed9071035a92b925eab037b24","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103c200\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-zg4fwpppdnrm\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9c0f6f2-b804-42ec-a112-a611d120c5aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9c0f6f2-b804-42ec-a112-a611d120c5aa\",\n \"name\": \"f9c0f6f2-b804-42ec-a112-a611d120c5aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:45:44.397135399Z\",\n \"endTime\": \"2026-09-17T03:14:49.364484388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00103c200\u003e: \n failed to create HCP cluster private-ingress: failed waiting for cluster=\"private-ingress\" in resourcegroup=\"private-ingress-zg4fwpppdnrm\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9c0f6f2-b804-42ec-a112-a611d120c5aa\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9c0f6f2-b804-42ec-a112-a611d120c5aa\",\n \"name\": \"f9c0f6f2-b804-42ec-a112-a611d120c5aa\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:45:44.397135399Z\",\n \"endTime\": \"2026-09-17T03:14:49.364484388Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m57s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e5779d4f7c41bfccbc89370f64e5fc8f6827e892c2cf596461baa5eda7cd4ef3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","failed_at":"e2e","signature_id":"a2ee109ec25c2b87338aedcebe4c8cc584f5a143bdc97ee69a387447d2c549b6","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb8740\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xqds56zcttv\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb2db9bc-c46e-47f3-966f-caceef2356cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb2db9bc-c46e-47f3-966f-caceef2356cf\",\n \"name\": \"bb2db9bc-c46e-47f3-966f-caceef2356cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:45:13.233682082Z\",\n \"endTime\": \"2026-09-17T03:14:19.374594663Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m6s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fb8740\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xqds56zcttv\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb2db9bc-c46e-47f3-966f-caceef2356cf\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bb2db9bc-c46e-47f3-966f-caceef2356cf\",\n \"name\": \"bb2db9bc-c46e-47f3-966f-caceef2356cf\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:45:13.233682082Z\",\n \"endTime\": \"2026-09-17T03:14:19.374594663Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 18m6s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":8,"failure_pattern_ids":["208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":8,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6948,"pr_state":"closed","pr_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","final_merged_sha":"95f6c7af1cc831fd35117e4cdcdb392c07e53217","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T00:46:16Z","started_at":"2026-09-17T00:46:16Z","completed_at":"2026-09-17T03:03:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:41:24.057791333Z","timing_metadata_checked_at":"2026-09-18T15:41:24.057791333Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:41:24.060808444Z","region_metadata_checked_at":"2026-09-18T15:41:24.060808444Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"24595d84d6473bf1b08d6f4e8443e8575033307296e004ecc3977be746f9c821","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","failed_at":"alert","signature_id":"e6916222b7c43fadfb77f5030602e037edec83d0ef773a05b13d08de9f0efc19","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 9 time(s) (1 unknown, 8 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T02:03:30Z\n Ended: 2026-09-17T02:34:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j2233600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2ssfjf9nv6ct1kgbo2ppvo99o553qc8o-v20260901\", pod=\"router-585848dd75-gbvhv\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2ssfjf9nv6ct1kgbo2ppvo99o553qc8o-v20260901/router-585848dd75-gbvhv has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"34c8dba76b148ee8273308bc8d25e1e49b322c377e4d9a6e91dd53ced6d59079","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","failed_at":"alert","signature_id":"a4314857c749da303216d314351149e0065b1d08354cb26b1fbedfc555fc6412","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 12 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:36:55Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-jxhxg\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-jxhxg (namespace arobit) on cluster ci01-j2233600-mgmt-2 has averaged 138.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n State: Fired (not resolved)\n Started: 2026-09-17T02:28:55Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-89c4fbff5-l2x67\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-89c4fbff5-l2x67 (namespace clusters-service) on cluster ci01-j2233600-svc has averaged 834.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:38:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-9wzdj\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-9wzdj (namespace arobit) on cluster ci01-j2233600-mgmt-1 has averaged 152.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:40:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-c4vts\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-c4vts (namespace arobit) on cluster ci01-j2233600-mgmt-2 has averaged 145.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:43:55Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-p8rsk\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-p8rsk (namespace arobit) on cluster ci01-j2233600-mgmt-1 has averaged 171.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n State: Fired (not resolved)\n Started: 2026-09-17T02:28:55Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-6b8ff47d7-pwxfw\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container aro-hcp-backend in pod aro-hcp-backend-6b8ff47d7-pwxfw (namespace aro-hcp) on cluster ci01-j2233600-svc has averaged 407.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:40:56Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5d4d9cb45c-fgrxt\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5d4d9cb45c-fgrxt (namespace mgmt-agent) on cluster ci01-j2233600-mgmt-2 has averaged 183.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 8:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:52:58Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-89c4fbff5-52qjp\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-89c4fbff5-52qjp (namespace clusters-service) on cluster ci01-j2233600-svc has averaged 291.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 9:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:42:57Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-cm5m2\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-cm5m2 (namespace arobit) on cluster ci01-j2233600-mgmt-2 has averaged 168.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 10:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:49:59Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-89c4fbff5-2h4r9\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container clusters-service-server in pod clusters-service-89c4fbff5-2h4r9 (namespace clusters-service) on cluster ci01-j2233600-svc has averaged 241.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 11:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:47:57Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-w8b66\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-w8b66 (namespace arobit) on cluster ci01-j2233600-svc has averaged 216.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 12:\n State: Resolved\n Started: 2026-09-17T02:28:55Z\n Ended: 2026-09-17T02:40:00Z\n Severity: Sev3\n Labels: alertname=\"ServiceCPUDrift\", cluster=\"ci01-j2233600-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-5d4d9cb45c-gh2st\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container mgmt-agent-controller in pod mgmt-agent-5d4d9cb45c-gh2st (namespace mgmt-agent) on cluster ci01-j2233600-mgmt-1 has averaged 173.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired"}},{"row_id":"78f1597dcadf96e5a188317204bd24dadfd1daab8f0626f03ac2da9573499fee","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","failed_at":"e2e","signature_id":"21b15360431929bdb593a2107c3896a1ca85c63523d6f4c20410fb74f623d0a9","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f81380\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cw4qg8klp5jk\" to finish creating: GET https://rp.j2233600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a8f0e67-43cc-40f1-a32a-616ff94b6ac1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a8f0e67-43cc-40f1-a32a-616ff94b6ac1\",\n \"name\": \"0a8f0e67-43cc-40f1-a32a-616ff94b6ac1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T01:51:55.906063696Z\",\n \"endTime\": \"2026-09-17T02:20:56.559080431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m43s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f81380\u003e: \n failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-cw4qg8klp5jk\" to finish creating: GET https://rp.j2233600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a8f0e67-43cc-40f1-a32a-616ff94b6ac1\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a8f0e67-43cc-40f1-a32a-616ff94b6ac1\",\n \"name\": \"0a8f0e67-43cc-40f1-a32a-616ff94b6ac1\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T01:51:55.906063696Z\",\n \"endTime\": \"2026-09-17T02:20:56.559080431Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m43s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"85f8e168cbf74e7d21a8e7924bde89286467e480ececcb425f0a064833307624","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","failed_at":"alert","signature_id":"54b42f0c1ae5cc077af2dfa797005520003ed2bd8ca931aabcdf79c00d97c263","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-17T02:28:29Z\n Ended: 2026-09-17T02:34:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2233600-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.195:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2233600/providers/microsoft.monitor/accounts/hcps-j2233600\", namespace=\"ocm-arohcpci01-2ssfjf9nv6ct1kgbo2ppvo99o553qc8o-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-gzzmc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ssfjf9nv6ct1kgbo2ppvo99o553qc8o-v20260901/router has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-17T02:28:29Z\n Ended: 2026-09-17T02:34:28Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j2233600-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.195:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2233600/providers/microsoft.monitor/accounts/hcps-j2233600\", namespace=\"ocm-arohcpci01-2ssfjf9nv6ct1kgbo2ppvo99o553qc8o-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-gzzmc\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2ssfjf9nv6ct1kgbo2ppvo99o553qc8o-v20260901/router has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100375512105881600","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6752,"pr_state":"closed","pr_sha":"d433c21668cfdc66f98262f9099c8086b4397248","final_merged_sha":"d433c21668cfdc66f98262f9099c8086b4397248","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T00:05:19Z","started_at":"2026-09-17T00:05:19Z","completed_at":"2026-09-17T00:05:56Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:40:20.494445626Z","timing_metadata_checked_at":"2026-09-18T15:40:20.494445626Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:40:20.497450337Z","region_metadata_checked_at":"2026-09-19T03:30:34.328005292Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"0cac1fdb89fd4e244f460aa49b373516db7a5acb6e9de771460d505f19bda2a4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100375512105881600","occurred_at":"2026-09-17T00:05:19Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}}]},{"environment":"int","summary":{"total_runs":8,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":1,"runs_with_unmatched_occurrences":4,"runs_without_occurrence_rows":3,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100715155292688384","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T22:34:57Z","started_at":"2026-09-17T22:34:57Z","completed_at":"2026-09-18T00:51:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:42.989600676Z","timing_metadata_checked_at":"2026-09-18T15:42:42.989600676Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:42.992661887Z","region_metadata_checked_at":"2026-09-18T15:42:42.992661887Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100631679885381632","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T17:03:14Z","started_at":"2026-09-17T17:03:14Z","completed_at":"2026-09-17T17:03:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:42.872507655Z","timing_metadata_checked_at":"2026-09-18T15:42:42.872507655Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:42:42.875838367Z","region_metadata_checked_at":"2026-09-19T03:30:51.273131973Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"f8f501d5c4aec76afdb008bc57f25d5ad6169d8b2d8dfb66223a2867a33ae869","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100631679885381632","occurred_at":"2026-09-17T17:03:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100629444883386368","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:54:22Z","started_at":"2026-09-17T16:54:22Z","completed_at":"2026-09-17T16:54:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:42.754715232Z","timing_metadata_checked_at":"2026-09-18T15:42:42.754715232Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:42:42.757846843Z","region_metadata_checked_at":"2026-09-19T03:30:51.176297118Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"62faa1825c1e2b465aaec9fc943ff5f23acaf3e5959a3faa4642887aae978559","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100629444883386368","occurred_at":"2026-09-17T16:54:22Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100619599048347648","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:15:14Z","started_at":"2026-09-17T16:15:14Z","completed_at":"2026-09-17T16:15:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:42.650822659Z","timing_metadata_checked_at":"2026-09-18T15:42:42.650822659Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:42:42.654361771Z","region_metadata_checked_at":"2026-09-19T03:30:50.954514506Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e41951d9827b65b15e309ac8cadc8e15350e63959f684fcbcee31e0f08dc2d22","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100619599048347648","occurred_at":"2026-09-17T16:15:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100616588343382016","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:03:16Z","started_at":"2026-09-17T16:03:16Z","completed_at":"2026-09-17T16:05:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:42.553025507Z","timing_metadata_checked_at":"2026-09-18T15:42:42.553025507Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:42:42.556170219Z","region_metadata_checked_at":"2026-09-19T03:30:50.862258369Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"17d14baff70787e20a697bbcb8bcf38d9d93ea39b63a2ff2b997f2aab0cccdd0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100616588343382016","occurred_at":"2026-09-17T16:03:16Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100532556209328128","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T10:29:22Z","started_at":"2026-09-17T10:29:22Z","completed_at":"2026-09-17T12:47:32Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:42.405945079Z","timing_metadata_checked_at":"2026-09-18T15:42:42.405945079Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:42.40912959Z","region_metadata_checked_at":"2026-09-18T15:42:42.40912959Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"2c4859fb4ff3bea14a70980db375f4bb1a8e35091374c99c6a17357492346058","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100532556209328128","occurred_at":"2026-09-17T10:29:22Z","failed_at":"e2e","signature_id":"a7b733023a299f40faa1bcb49b8265ed28d3f85ddfbf335757885b57a1e7d20f","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:162]: failed to verify HCP cluster ephemeral-disk is viable\nUnexpected error:\n \u003c*errors.joinError | 0xc0017886a8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:162]: failed to verify HCP cluster ephemeral-disk is viable\nUnexpected error:\n \u003c*errors.joinError | 0xc0017886a8\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"44813a24efc9a4e87de23a1d03f218de3f44f990811139bedf70528921d1e908","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout"}},{"row_id":"4eed197cfaa3c759400f4686499114f6ad6e2de83f48a8c3a9a7c389aad5be05","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100532556209328128","occurred_at":"2026-09-17T10:29:22Z","failed_at":"e2e","signature_id":"06051e7fd69c31d191475d70ad084a2f5f338204d9dcb2ccb5a0566c0fc34ac7","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","test_suite":"integration/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:612]: failed to get node release images before downgrade\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019a0220\u003e: \n list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:612]: failed to get node release images before downgrade\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0019a0220\u003e: \n list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"221cb0b46f3bef91cff2ee06c068eca8fb3646ef7eeb7ed9cd39ae59c1446e81","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["221cb0b46f3bef91cff2ee06c068eca8fb3646ef7eeb7ed9cd39ae59c1446e81","44813a24efc9a4e87de23a1d03f218de3f44f990811139bedf70528921d1e908"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100444402584391680","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T04:39:04Z","started_at":"2026-09-17T04:39:04Z","completed_at":"2026-09-17T06:57:53Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:42.290173563Z","timing_metadata_checked_at":"2026-09-18T15:42:42.290173563Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:42.294034677Z","region_metadata_checked_at":"2026-09-18T15:42:42.294034677Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"int","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100386878376120320","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T00:50:29Z","started_at":"2026-09-17T00:50:29Z","completed_at":"2026-09-17T02:53:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:42.146981249Z","timing_metadata_checked_at":"2026-09-18T15:42:42.146981249Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:42.15026346Z","region_metadata_checked_at":"2026-09-18T15:42:42.15026346Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]},{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"runs_with_occurrences":5,"runs_with_failure_pattern_matches":3,"runs_with_unmatched_occurrences":2,"runs_without_occurrence_rows":0,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T23:00:17Z","started_at":"2026-09-17T23:00:17Z","completed_at":"2026-09-18T00:46:11Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:49.01738054Z","timing_metadata_checked_at":"2026-09-18T15:42:49.01738054Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:49.020950553Z","region_metadata_checked_at":"2026-09-18T15:42:49.020950553Z"},"failed_at":["e2e"],"failed_test_count":12,"occurrences":[{"row_id":"16f33618910d894e50c047036449fc9317f05e0c13228e36eb5729f75d31928d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"a0e7d58faa2918f496570ed9c5f40cc4898c7cc2d361ef4097b193762f74fdcc","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cv4zb86lrk99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f21140\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cv4zb86lrk99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cv4zb86lrk99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n \u003c*exported.ResponseError | 0xc000f21140\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cv4zb86lrk99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: SubscriptionNotRegistered\n --------------------------------------------------------------------------------\n {\n \"code\": \"SubscriptionNotRegistered\",\n \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered"}},{"row_id":"1fabb8b2e0f17e512284051b11cb1927b2ed74f54aa8b177a8c02483d6d2e88f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"ae583f4fcd37f743d2e4acb91167ea667e50e9c13e01db10c235ffe6dd99caa7","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h9khdk\" for cluster \"cluster-nightly-4-20-h9khdk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adc920\u003e: \n failed to create NodePool np-h9khdk: failed starting nodepool creation \"np-h9khdk\" for cluster \"cluster-nightly-4-20-h9khdk\" in resourcegroup=\"rg-nightly-4-20-9v8vn2pfzhs7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9v8vn2pfzhs7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-h9khdk/nodePools/np-h9khdk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-h9khdk\" for cluster \"cluster-nightly-4-20-h9khdk\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000adc920\u003e: \n failed to create NodePool np-h9khdk: failed starting nodepool creation \"np-h9khdk\" for cluster \"cluster-nightly-4-20-h9khdk\" in resourcegroup=\"rg-nightly-4-20-9v8vn2pfzhs7\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-9v8vn2pfzhs7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-h9khdk/nodePools/np-h9khdk\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"892805971d889a7d7ee1aaaaa788b7709921d3b90a09c95784ef4ca5b5cc608f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"eedd7ed28a2e498144c6499476b15f9b62b521f665937a7144186b5981a0ab86","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00079eb40\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-8mb2dp88brbx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-8mb2dp88brbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00079eb40\u003e: \n failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-8mb2dp88brbx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-8mb2dp88brbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"8d742fe4fd7980bb3bc069770562dba2ff6d0642bd497e641e4f7006275b28a0","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"475d18438d6129ee057302e14cb4929cac287683cc42b505124692ed9000396d","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000996700\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r7m2fk\" in resourcegroup=\"rg-np-version-upgrade-r7m2fk-5jlmq454vn2n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7m2fk-5jlmq454vn2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r7m2fk/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000996700\u003e: \n failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r7m2fk\" in resourcegroup=\"rg-np-version-upgrade-r7m2fk-5jlmq454vn2n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7m2fk-5jlmq454vn2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r7m2fk/nodePools/npupgrade-4-20\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"9071a7958479384dec11c7aae6a0d932e45629d97b24f06152d0577294188342","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"5b8b71dd55429c7ea7e12ce3e8d437a79ae271abc6b3fd1cf66b2d542a2f54ac","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001466840\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-psjtgg\" resourcegroup=\"rg-np-dg-minor-psjtgg-2t4zvgfl9nx7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-psjtgg-2t4zvgfl9nx7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-psjtgg/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001466840\u003e: \n failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-psjtgg\" resourcegroup=\"rg-np-dg-minor-psjtgg-2t4zvgfl9nx7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-psjtgg-2t4zvgfl9nx7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-psjtgg/nodePools/npdg-422-420\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a40e23f80790d24548cdafd08351f7c83fcaf52a23ff704359afee67ddd5ec28","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"19d9edd34f8e48701cd8bbda924191e80690cc1893c6e47aa7458abc045a4095","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0020c7110\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-9jm8bqptv2wt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0020c7110\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-9jm8bqptv2wt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"a554de292e8efe39b5fdb17e41a1f8390e750bc910d2da9f5d94c4840d168c3a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"d144fcda6ced4631fcc19c7abd504ffe793c4fffbba643dd7b8a0d80f55c67e1","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014a8de0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-pzlb2tkmb5c5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0014a8de0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-pzlb2tkmb5c5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"b51f79b56e64fceb89add9cbc1da7e04db12283d919b339eb0bd6f725dfe8dca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"8af91b6a912b93a55a51953f18f51b6e88bc18db738006d3b71d9e5d820486c4","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00062e0a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-khvwxc7hm457\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9a805e6d-f4c1-448f-ac8d-eb4629159049\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9a805e6d-f4c1-448f-ac8d-eb4629159049\",\n \"name\": \"9a805e6d-f4c1-448f-ac8d-eb4629159049\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:14:40.679038392Z\",\n \"endTime\": \"2026-09-17T23:33:46.186414821Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00062e0a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-khvwxc7hm457\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9a805e6d-f4c1-448f-ac8d-eb4629159049\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9a805e6d-f4c1-448f-ac8d-eb4629159049\",\n \"name\": \"9a805e6d-f4c1-448f-ac8d-eb4629159049\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T23:14:40.679038392Z\",\n \"endTime\": \"2026-09-17T23:33:46.186414821Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"d72f64d1dce1180ba19e7342b3fb5b40ebd5126dde3929cd3e247c820fbacabc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"787555a75756b0bcd3c50e5022f58b6e6c990b64d6f87d9979ae97767e7916f5","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000fe8ae0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n \u003c*errors.joinError | 0xc000fe8ae0\u003e: \n VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n ...","failure_pattern_match":{"status":"clustered","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)"}},{"row_id":"eea9c41bc9c6cd2f70b67074cb51f98e607964f30bcd5f633b7bdbf5a156eefe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"b64e2e209238a4291f7bc6ca875c4002b2b72ce61c22b00cc341eff1d3d7d329","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013e8bd0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-n8nkjmlflmx2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n \u003c*exported.ResponseError | 0xc0013e8bd0\u003e: \n PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-n8nkjmlflmx2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f29494c4f0b35ac292959b2020f71a7683ab3f2f002ca93780fdb48a0799dabe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"9f76555451648e830890a7c9cd43251ffcd8ea938c9675715ab08891b03af525","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007184c0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-cqdwqx\" in resourcegroup=\"rg-np-skip-minor-cqdwqx-w27zwxzrpszc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-cqdwqx-w27zwxzrpszc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-cqdwqx/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0007184c0\u003e: \n failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-cqdwqx\" in resourcegroup=\"rg-np-skip-minor-cqdwqx-w27zwxzrpszc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-cqdwqx-w27zwxzrpszc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-cqdwqx/nodePools/nps-420-422\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}},{"row_id":"f89ce9184a45813940fd8da8f4cbbdedf5739a65ad815d402bd9793dee2321da","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","failed_at":"e2e","signature_id":"ad4219e21e3f54d13de0b5a81134597e91fe66a1442cf0f6968126d652406ccf","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002b70300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-9mx94tspcdqp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n \u003c*exported.ResponseError | 0xc002b70300\u003e: \n PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-9mx94tspcdqp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n --------------------------------------------------------------------------------\n RESPONSE 400: 400 Bad Request\n ERROR CODE: InvalidRequestContent\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InvalidRequestContent\",\n \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n \"target\": \"properties.version.id\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent"}}],"failure_pattern_summary":{"signature_count":12,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282"],"matched_occurrences":12,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T21:11:04Z","started_at":"2026-09-17T21:11:04Z","completed_at":"2026-09-17T23:37:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:47.693772282Z","timing_metadata_checked_at":"2026-09-18T15:42:47.693772282Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:47.697888497Z","region_metadata_checked_at":"2026-09-18T15:42:47.697888497Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"93fd0e1b949e9bb3a047b955a361a0d17aa719a7e6649db63de787cc2771b559","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","failed_at":"e2e","signature_id":"be41217a9374881e99f1a9416ccadf32d5a5da3fe931b3a6439f304be981164d","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002740c00\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5ddg8tcqt24f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a27ca6a0-c950-4180-9609-7a9ade0dfaf2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a27ca6a0-c950-4180-9609-7a9ade0dfaf2\",\n \"name\": \"a27ca6a0-c950-4180-9609-7a9ade0dfaf2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:32:08.724911504Z\",\n \"endTime\": \"2026-09-17T22:51:12.585683878Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n \u003c*fmt.wrapError | 0xc002740c00\u003e: \n failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5ddg8tcqt24f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a27ca6a0-c950-4180-9609-7a9ade0dfaf2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a27ca6a0-c950-4180-9609-7a9ade0dfaf2\",\n \"name\": \"a27ca6a0-c950-4180-9609-7a9ade0dfaf2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:32:08.724911504Z\",\n \"endTime\": \"2026-09-17T22:51:12.585683878Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"bb03d813b6e2210593c894275b12326d751232679c228e1a673b798bbe49de03","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","failed_at":"e2e","signature_id":"ce783312029a03f3c6f81cf899cc96d5f27f953623a9a5fa1b6b579e78455f50","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a4edc0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-rrzr5s\" in resourcegroup=\"rg-np-noedge-rrzr5s-tw6bdclngxdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e383065-95e4-447a-8c07-f8b7ac95870b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e383065-95e4-447a-8c07-f8b7ac95870b\",\n \"name\": \"4e383065-95e4-447a-8c07-f8b7ac95870b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:45:09.171165434Z\",\n \"endTime\": \"2026-09-17T23:04:12.617623397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a4edc0\u003e: \n failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-rrzr5s\" in resourcegroup=\"rg-np-noedge-rrzr5s-tw6bdclngxdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e383065-95e4-447a-8c07-f8b7ac95870b\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e383065-95e4-447a-8c07-f8b7ac95870b\",\n \"name\": \"4e383065-95e4-447a-8c07-f8b7ac95870b\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:45:09.171165434Z\",\n \"endTime\": \"2026-09-17T23:04:12.617623397Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e3889b7985757a3d0ef89c157586f8c3a24e7497ee591662a870f59b443d242f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","failed_at":"e2e","signature_id":"e9d197f55b7c8094b6f94fd2ac343ddfe7d54231e67ed0c0282b273041aea3b6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-rwd7s5\" for cluster \"cluster-candidate-4-20-rwd7s5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014fd540\u003e: \n failed to create NodePool np-rwd7s5: failed waiting for nodepool=\"np-rwd7s5\" for cluster \"cluster-candidate-4-20-rwd7s5\" in resourcegroup=\"rg-candidate-4-20-dgf456bm6cvc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5770130-02b8-4165-803f-f4f44190a860\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5770130-02b8-4165-803f-f4f44190a860\",\n \"name\": \"a5770130-02b8-4165-803f-f4f44190a860\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:08:49.966569897Z\",\n \"endTime\": \"2026-09-17T22:27:49.362137457Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-rwd7s5\" for cluster \"cluster-candidate-4-20-rwd7s5\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0014fd540\u003e: \n failed to create NodePool np-rwd7s5: failed waiting for nodepool=\"np-rwd7s5\" for cluster \"cluster-candidate-4-20-rwd7s5\" in resourcegroup=\"rg-candidate-4-20-dgf456bm6cvc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5770130-02b8-4165-803f-f4f44190a860\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a5770130-02b8-4165-803f-f4f44190a860\",\n \"name\": \"a5770130-02b8-4165-803f-f4f44190a860\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T22:08:49.966569897Z\",\n \"endTime\": \"2026-09-17T22:27:49.362137457Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100619096168075264","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:13:14Z","started_at":"2026-09-17T16:13:14Z","completed_at":"2026-09-17T16:13:33Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:47.564471717Z","timing_metadata_checked_at":"2026-09-18T15:42:47.564471717Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:42:47.567794829Z","region_metadata_checked_at":"2026-09-19T03:30:51.738254475Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"752027b0fee3db38286301469f8cc825e931ee31f6be741919008d69b466fb56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100619096168075264","occurred_at":"2026-09-17T16:13:14Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100616789380567040","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T16:04:04Z","started_at":"2026-09-17T16:04:04Z","completed_at":"2026-09-17T16:06:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:47.458573237Z","timing_metadata_checked_at":"2026-09-18T15:42:47.458573237Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-09-18T15:42:47.461671148Z","region_metadata_checked_at":"2026-09-19T03:30:51.640565818Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"ff6ebf94dcb38abaead0020f0e1b6c8f8546eb09d4bbf95c803184b135645838","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100616789380567040","occurred_at":"2026-09-17T16:04:04Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"prod","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100404440946184192","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T02:00:16Z","started_at":"2026-09-17T02:00:16Z","completed_at":"2026-09-17T04:05:14Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:49.203898111Z","timing_metadata_checked_at":"2026-09-18T15:42:49.203898111Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:49.21755656Z","region_metadata_checked_at":"2026-09-18T15:42:49.21755656Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"1a638cfaeb9607d2dc61a65b91a25f86c4061bde5eddd23f656df0b8bd3e9774","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100404440946184192","occurred_at":"2026-09-17T02:00:16Z","failed_at":"e2e","signature_id":"ce1e7db080deea06cb7610e1f4b4bf939ebd4afb01e8c1a7763c9d9cd1a9bd32","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","test_suite":"prod/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ddaa80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-mkcwd2qnbq2q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eef2b4be-3845-4db8-8a9e-a82745bc8bcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eef2b4be-3845-4db8-8a9e-a82745bc8bcd\",\n \"name\": \"eef2b4be-3845-4db8-8a9e-a82745bc8bcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:13:07.43461625Z\",\n \"endTime\": \"2026-09-17T02:32:13.121368333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:139]: failed to create nodepool \"np-1\" for autoscaling cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ddaa80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-mkcwd2qnbq2q\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eef2b4be-3845-4db8-8a9e-a82745bc8bcd\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/eef2b4be-3845-4db8-8a9e-a82745bc8bcd\",\n \"name\": \"eef2b4be-3845-4db8-8a9e-a82745bc8bcd\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T02:13:07.43461625Z\",\n \"endTime\": \"2026-09-17T02:32:13.121368333Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":3,"runs_with_occurrences":3,"runs_with_failure_pattern_matches":2,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":1,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100721533742747648","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T23:00:17Z","started_at":"2026-09-17T23:00:17Z","completed_at":"2026-09-18T00:02:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:55.948782246Z","timing_metadata_checked_at":"2026-09-18T15:42:55.948782246Z","region_metadata_state":"invalid","region_metadata_first_checked_at":"2026-09-18T15:42:55.951883757Z","region_metadata_checked_at":"2026-09-19T03:30:53.102436969Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"e8d4e8b773b4055d0a4446558c0cf426c3cbc133735d0203658203f2b6027861","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2100721533742747648","occurred_at":"2026-09-17T23:00:17Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100534031866138624","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T10:35:13Z","started_at":"2026-09-17T10:35:13Z","completed_at":"2026-09-17T12:47:34Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:55.67145025Z","timing_metadata_checked_at":"2026-09-18T15:42:55.67145025Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:55.674716761Z","region_metadata_checked_at":"2026-09-18T15:42:55.674716761Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"08761405fc21ed84f3653028d03d67f4c6efb0d14ed7b120b8375caed90d75b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100534031866138624","occurred_at":"2026-09-17T10:35:13Z","failed_at":"e2e","signature_id":"aac82bf00f815d526a50ddb15d7fc450980f914b55421ea2243e84387908cb3a","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-4-22-j5vkzr\" is viable before upgrade\nUnexpected error:\n \u003c*errors.joinError | 0xc0015d2858\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-4-22-j5vkzr\" is viable before upgrade\nUnexpected error:\n \u003c*errors.joinError | 0xc0015d2858\u003e: \n VerifyBasicAccess failed: failed to list services: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9f24424c672066cba3c7cb45f333ebe45b01b356fab007f5a65bee0f062d44cf","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9f24424c672066cba3c7cb45f333ebe45b01b356fab007f5a65bee0f062d44cf"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-09-17T05:11:15Z","started_at":"2026-09-17T05:11:15Z","completed_at":"2026-09-17T07:24:26Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:55.536380965Z","timing_metadata_checked_at":"2026-09-18T15:42:55.536380965Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:55.541439583Z","region_metadata_checked_at":"2026-09-18T15:42:55.541439583Z"},"failed_at":["e2e"],"failed_test_count":3,"occurrences":[{"row_id":"81240792535ea1455a9080d8e75ab82e6ca9791f6f59c2b5263349193afdf727","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","failed_at":"e2e","signature_id":"cc190a2c1395e4f5ad1b156f9318c22909f9bd03dbf07b404cb1ad51d8dfd21e","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0035265a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-rcr2wwpjvcjv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n \"name\": \"5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T05:39:36.012772677Z\",\n \"endTime\": \"2026-09-17T05:58:42.168749593Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0035265a0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-rcr2wwpjvcjv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n \"name\": \"5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T05:39:36.012772677Z\",\n \"endTime\": \"2026-09-17T05:58:42.168749593Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"94fcd42bae1c31838b5fc43246102194a202628bcd2ff177bbcbb4669c5e0c01","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","failed_at":"e2e","signature_id":"573bf63f25f718e6e10d14bc2ae7830ca909cb9577c72529929dab5db0a4d7ee","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc2640\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xfvdn4xsffw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\",\n \"name\": \"98b08bb9-0eee-4840-9535-91464d8458a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T05:40:34.02322608Z\",\n \"endTime\": \"2026-09-17T05:59:42.272250066Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fc2640\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xfvdn4xsffw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\",\n \"name\": \"98b08bb9-0eee-4840-9535-91464d8458a2\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T05:40:34.02322608Z\",\n \"endTime\": \"2026-09-17T05:59:42.272250066Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}},{"row_id":"e6b63641ea3e940f397530d60dc213e05c8ca9c4677dea1a6746cf9785cd16fc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","failed_at":"e2e","signature_id":"d97b1d417fbed487ffa515ee9e94f38afd2685c1f5cc55df9fad23cc5afc6bc9","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","test_suite":"stage/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-72xhwq\" for cluster \"cluster-candidate-4-22-72xhwq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009bb400\u003e: \n failed to create NodePool np-72xhwq: failed waiting for nodepool=\"np-72xhwq\" for cluster \"cluster-candidate-4-22-72xhwq\" in resourcegroup=\"rg-candidate-4-22-ffxlnxmpk4tb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ee3e506-9091-4aa0-ad2c-045d3e78731e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ee3e506-9091-4aa0-ad2c-045d3e78731e\",\n \"name\": \"8ee3e506-9091-4aa0-ad2c-045d3e78731e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T06:05:40.036081638Z\",\n \"endTime\": \"2026-09-17T06:24:44.492140979Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-72xhwq\" for cluster \"cluster-candidate-4-22-72xhwq\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0009bb400\u003e: \n failed to create NodePool np-72xhwq: failed waiting for nodepool=\"np-72xhwq\" for cluster \"cluster-candidate-4-22-72xhwq\" in resourcegroup=\"rg-candidate-4-22-ffxlnxmpk4tb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ee3e506-9091-4aa0-ad2c-045d3e78731e\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ee3e506-9091-4aa0-ad2c-045d3e78731e\",\n \"name\": \"8ee3e506-9091-4aa0-ad2c-045d3e78731e\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-17T06:05:40.036081638Z\",\n \"endTime\": \"2026-09-17T06:24:44.492140979Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError"}}],"failure_pattern_summary":{"signature_count":3,"failure_pattern_ids":["d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f"],"matched_occurrences":3,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"stg","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2100404441030070272","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","pr_number":0,"pr_state":"","pr_sha":"","final_merged_sha":"","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-09-17T02:00:16Z","started_at":"2026-09-17T02:00:16Z","completed_at":"2026-09-17T03:56:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-09-18T15:42:56.079507615Z","timing_metadata_checked_at":"2026-09-18T15:42:56.079507615Z","region":"uksouth","region_metadata_state":"found","region_metadata_first_checked_at":"2026-09-18T15:42:56.082953227Z","region_metadata_checked_at":"2026-09-18T15:42:56.082953227Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}