{"meta":{"start_date":"2026-09-20","end_date":"2026-09-20","start_at":"2026-09-20T00:00:00Z","end_at":"2026-09-21T00:00:00Z","timezone":"UTC","generated_at":"2026-10-29T10:15:02Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":36,"failed_runs":30,"raw_occurrences":124,"matched_occurrences":121,"runs_affected":27},"rows":[{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":27,"runs_affected":15,"failed_runs":15,"run_impact_percent":41.666666666666664,"anchor_occurrences":27,"after_last_push_count":20,"trend_counts":[0,0,22,19,1,1,27],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":29,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":14},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00139c1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kctnzvxjhvcd\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"name\": \"3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:02:07.101805352Z\",\n \"endTime\": \"2026-09-21T01:26:13.044715763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2i1qqe304gugno1vb23hj27er6drc still exists (deletion dispatched at 2026-09-21T01:02:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00139c1f8\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-kctnzvxjhvcd\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"name\": \"3d51f09c-5120-4516-b9ee-a66be3d59304\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:02:07.101805352Z\",\n \"endTime\": \"2026-09-21T01:26:13.044715763Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2i1qqe304gugno1vb23hj27er6drc still exists (deletion dispatched at 2026-09-21T01:02:08Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001838090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s2nb9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-2wc8bnqpjvsw\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"name\": \"3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:41.012625795Z\",\n \"endTime\": \"2026-09-21T01:06:41.448864721Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2hu8mbpcvgabffnu4e7eeji5v72ts still exists (deletion dispatched at 2026-09-21T00:42:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001838090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-21-s2nb9c\" in resourcegroup=\"rg-zstream-upgrade-4-21-2wc8bnqpjvsw\" to finish deleting: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"name\": \"3e7690c3-5896-4508-ab34-c5babe08c719\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:41.012625795Z\",\n \"endTime\": \"2026-09-21T01:06:41.448864721Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2hu8mbpcvgabffnu4e7eeji5v72ts still exists (deletion dispatched at 2026-09-21T00:42:44Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00174e0c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k6sk5ztcb6st\" to finish deleting: GET https://rp.j7913600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"name\": \"2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:34.467872497Z\",\n \"endTime\": \"2026-09-21T01:06:34.900855786Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2gfj01vbki6vfv5k1lnt4j81kjqnb still exists (deletion dispatched at 2026-09-21T00:42:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc00174e0c0\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-k6sk5ztcb6st\" to finish deleting: GET https://rp.j7913600.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"name\": \"2aeb64dc-0a85-4998-9985-55bcafbd5fc4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:42:34.467872497Z\",\n \"endTime\": \"2026-09-21T01:06:34.900855786Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2gfj01vbki6vfv5k1lnt4j81kjqnb still exists (deletion dispatched at 2026-09-21T00:42:43Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"48429fa1e1b278e83c9c1235be2155e15cb94e91feb6a91189f07417aac2381b","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"ae6ec300b2e1b5ec89e4505b36ba0351611d82380af69c6bc2aa2a3220759883","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","signature_id":"e4aee9aff864d2f43ab51682459e47fa9b2962afca634816bc039aba490af5e3","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","occurred_at":"2026-09-20T20:15:04Z","signature_id":"55fec78f783ce8cf181e98d25b36b712ff6eab60ce5978dd2cbcd0aee43a639c","pr_number":7033,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101767145149960192","occurred_at":"2026-09-20T20:15:04Z","signature_id":"753848c3bc300eb5da8ad64aa08adc3a8b8dafee7fb505106b6f5cbf501ef653","pr_number":7033,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"1de82b4d07b3d460f0773709ee7a723eb163f891926727a6696a1f16b19d90b5","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"35c9e2930b67189ab544a31c2246d603785a353beb1727154337c02311b5ba68","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"6b62ee5741da84d36547bbd46f0d60714cb4fa8627bb19d966f92a3eba03e28a","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"6ecc7b9cabb4ec501df9af223046a31e6f46790bc97afea6159cc8363fead315","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"ae135f0135bd68e7f6075a32b2c72f3bdce2c27a9f86f02269bf9ec0d9c29b8b","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"5e8455d974423d74da7a2764a6de9171d767628d73b6837a9e7152a9dc31c031","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"94c88bfe25b6cc5073858901db97efc9bfdacd7a00fdfffa45d0abf7a6f10cff","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"ba4ec05dffbf7aeb4b673f4a633bc6bb64ed25fa0b09ea36f4467a08976b4236","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"6430a6d078ca6a6f372c5a118c728de07a27f33f7e2dd5e2fdf9a7c60c12d6f7","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"81d3fa9d4a49bf594db48e69cfb405302d6d567f873fa4a49e7a724e1f6a4553","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"db98f57638dcde7ae6924500ef05678fc197bea0143f095eee401504ac389cb8","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"66049a9e3aec5154c649a5163e0d3ee1db514ce22e0071326628d432e4a129e1","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","occurred_at":"2026-09-20T16:49:39Z","signature_id":"c01cceb55dbabd6acf2c61e2aecc8f6bbf9895f9ca156a63db239382788d9d1e","pr_number":7033,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","occurred_at":"2026-09-20T16:24:06Z","signature_id":"7a7b14873ebc7e928a8df3faa64a26b8a0c2ace4736d3d613f6a89c5b2c7ebba","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101708991846420480","occurred_at":"2026-09-20T16:24:06Z","signature_id":"f680e9c6bf353ff7e3737b28ae5ee7746498d1917182a5b8a67f475e211714a3","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","signature_id":"2eff4fde8ffd9f3247fb66660611f250c208eb3abb98228934a36763af0a55d7","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","signature_id":"b369fa1435784bfd120c7576bed94a6e4a39fcf725052260c677eef0e62863e5","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101670587393904640","occurred_at":"2026-09-20T13:51:29Z","signature_id":"10316e5a87c9726ae96606bd5b16607ddf7b0d400689c8e8c0e050115901ad56","pr_number":7026,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7029/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643457066962944","occurred_at":"2026-09-20T12:03:41Z","signature_id":"5fb2a2918cf998e33798a60ea71579a7a8d626d7c294f1e7d5559e93c8cb6517","pr_number":7029,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","occurred_at":"2026-09-20T11:27:46Z","signature_id":"b854729ec479cb89d9449d9a6d7d41e7ad976014ffc19decf951194f7d9ca05c","pr_number":7026,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610157367627776","occurred_at":"2026-09-20T09:51:22Z","signature_id":"d3228063fdb4db7077f94a323b61ddb40a6e50e0a6a4ce81d902c823eb177dc2","pr_number":7025,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7023/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101588887028633600","occurred_at":"2026-09-20T08:26:50Z","signature_id":"3a8359bef606bfcfc9ffd99f5dc07bf9b62993cb12fd5d570d46aa5683d99f29","pr_number":7023,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":22,"runs_affected":4,"failed_runs":4,"run_impact_percent":11.11111111111111,"anchor_occurrences":22,"after_last_push_count":30,"trend_counts":[0,0,7,1,3,1,22],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":16,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa01c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-rsgvv4vlx279\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"name\": \"f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:58.238504906Z\",\n \"endTime\": \"2026-09-21T00:56:02.680818108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:124]: failed to create HCP cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000fa01c0\u003e: \n failed to create HCP cluster ingress-cert-le: failed waiting for cluster=\"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-rsgvv4vlx279\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"name\": \"f9fec3e4-c9fa-4d6e-bf26-76cfce3e43c6\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:26:58.238504906Z\",\n \"endTime\": \"2026-09-21T00:56:02.680818108Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006001c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nzxl7wns7x4f\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"name\": \"5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:09.310854698Z\",\n \"endTime\": \"2026-09-21T00:56:12.748107548Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0006001c0\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-nzxl7wns7x4f\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"name\": \"5b2c0231-f920-4a27-986e-a3e4ef41850c\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:09.310854698Z\",\n \"endTime\": \"2026-09-21T00:56:12.748107548Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001226220\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ctqghmzsdxw5\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"name\": \"ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:10.309010209Z\",\n \"endTime\": \"2026-09-21T00:56:12.764905068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n \u003c*fmt.wrapError | 0xc001226220\u003e: \n failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-ctqghmzsdxw5\" to finish creating: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"name\": \"ed9a0c72-7da1-479d-80fd-6a53427b28d7\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:10.309010209Z\",\n \"endTime\": \"2026-09-21T00:56:12.764905068Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"0a72c2fe9bb203f971f92d8e51c30ced5c7cc31b87aab0f0b5be2aa7c9918b6a","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"1d7e40ff9457b0a0fd1f510836392b2a61ea6d2ec70f11c154614ed442a91215","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"36da18cbc5bd0fcbf1bcfacf064639d80f3a1a5cf27962716b02e77a1bbfec85","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"3e0062883f6a7b1aa6861d60c4689b05572197b0ccbb7b68cf2a6bb313f310ed","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"4566340774f4d5644a077677c8965bc9db5a33855b406b923890c6903dd9241c","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"5f4073b27d81b6ba979875f303e0026e5d88b19010e94d8fbda6cdd43405762d","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"7423340e196d990fa25ef185f5f19b2603dd00ed381a561dc6f7f6a62639d096","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"86240fd1d6cc9a1cfec477b60417613aecc1fa5308bed9e0b396aa7862dc467a","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"8e12d0852cea0a6ef4c2f62124f08fedad1faab413533a1887ee7ffe0186f31d","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"d6906ab33d96cae1ef1353787751cc0c7d0fd1b9e2f1ba9fdecffdd4391cd20a","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"e126abbe3fd20bf154b2a5a83d9a532f73d8c15de2b02bdae2fe166758e207a1","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"fb228348ca1cd58b436b3544af73f3d69eabd197936f9379767b0fcf28b2b9e2","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"eac5055c826c77b923258da473791d4fdf5723c4be718ddb37bd909244aa07ad","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"0122ecc6da0d7f29528efc4c03aacfe2a95abd22053ee427b3ba034f89f24578","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"6bc7534690e6bb03e8cc5f05e1f2bfee87b13f961ed915d568dd8e6b92f0e0f2","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"7486390772235b7b4ad872f03600862b7205a72c98df5cd28e716390fedf267b","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"a2be2cf8041303505466850a3cb9c21d1f99bbb048308b395ef9642bc9197ca2","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"a682bdcc6ac5308107646bacc01ed06ff73822f3817c828a7af6eeb2d50dbb37","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"aea772e72e8fe85891b43563346768f4449021077f900a91ce91ffe2eb62aef3","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"bf5a3ba4ae1888ec5bbf4c1ea6088363a29b33944bb08384b179d0e7edc222cb","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"c4e33f88c11b927f53f19ef5f88b383cbdc6921ffa0d1b5ed16a016693d69ee1","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","signature_id":"21b7ce7b54d77195fc617e821329b90e750c10192c6be2fdd3f4089b6662713d","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fbb6166cd32229449dffd7ed566f4dd5c12bffa274bbfb52305b5dd844b57504","failure_pattern":"ERROR CODE: DeploymentFailed; detail code InternalServerError; detail message An error occurred.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":20,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":20,"after_last_push_count":20,"trend_counts":[0,0,0,0,0,0,20],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:76]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02d60\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kas-m75gws\" in resourcegroup=\"private-kas-s98lp67jv589\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-kas-s98lp67jv589/providers/Microsoft.Resources/deployments/customer-infra-private-kas-m75gws/operationStatuses/08584116992457285882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:76]: failed to create customer resources for private KAS cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f02d60\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-kas-m75gws\" in resourcegroup=\"private-kas-s98lp67jv589\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-kas-s98lp67jv589/providers/Microsoft.Resources/deployments/customer-infra-private-kas-m75gws/operationStatuses/08584116992457285882\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a11f20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-ltthln\" in resourcegroup=\"private-ingress-6mv7ddflq9wr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-6mv7ddflq9wr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-ltthln/operationStatuses/08584116992436980878\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:77]: failed to create customer resources for private ingress cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a11f20\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-private-ingress-ltthln\" in resourcegroup=\"private-ingress-6mv7ddflq9wr\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/private-ingress-6mv7ddflq9wr/providers/Microsoft.Resources/deployments/customer-infra-private-ingress-ltthln/operationStatuses/08584116992436980878\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:110]: failed to create customer resources for cluster-subnet-reuse\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12720\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-h6k9tb\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-h6k9tb/operationStatuses/08584116992459228470\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:110]: failed to create customer resources for cluster-subnet-reuse\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000c12720\u003e: \n failed to create customer-infra: failed waiting for deployment \"customer-infra-cluster-subnet-reuse-h6k9tb\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-cluster-nsg-subnet-reuse-xf4hx8gcmq7v/providers/Microsoft.Resources/deployments/customer-infra-cluster-subnet-reuse-h6k9tb/operationStatuses/08584116992459228470\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: DeploymentFailed\n --------------------------------------------------------------------------------\n {\n \"status\": \"Failed\",\n \"error\": {\n \"code\": \"DeploymentFailed\",\n \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n \"details\": [\n {\n \"code\": \"Conflict\",\n \"message\": \"{\\r\\n \\\"status\\\": \\\"Failed\\\",\\r\\n \\\"error\\\": {\\r\\n \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n \\\"details\\\": [\\r\\n {\\r\\n \\\"code\\\": \\\"InternalServerError\\\",\\r\\n \\\"message\\\": \\\"An error occurred.\\\",\\r\\n \\\"details\\\": []\\r\\n }\\r\\n ]\\r\\n }\\r\\n}\"\n }\n ]\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"1a79c558038cb73ff72359e020fc0d6f9f740e31cbd0af1daebaf91ad6c65fa3","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"1b003937044cfa4fcd5a2675364491c0f6ffb7db83c6398081a44e0f34d6bb6b","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"268106fe2eb57086c4eef48987e800efaac560af7778223cb42b3520a44d6749","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"2b974162c7e6d97b0de4aea836205f4f789664585153978c18ab76f65fd6ff59","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"42440a2d7bb764d5edcdcdcd65e32a762540470eaa9f7ce1559f0c2a375c81a0","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"454410db4b0c9c7e89e9e307d8cabd3be3a5cc6c620a919683872059242695a3","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"56478c58658652b4bc1945a590ac05ad32b42a6f22c2e138039cb7ee3c4cbfac","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"5d0ca65ae867bd15279982b3a25e60bccfc658f662dfee162bb114f2ddaccb2c","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"73c41b364dee2fd3eb27b4ccaa57fb601deed774cfcc5111654a86e31aa1c0ef","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"75b9233e5419e103c738cc032e9a0d4380e13a4936bd5bad2e51413dae0bf26f","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"81b579506962c7694460b4e0ba3269306f93397ebe18eb85545cae4066854205","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"91a719e5ea3aea8b9b0e188fedf3ccfcff48a4a62dc5fc1afe26e222804d6227","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"9db4a31a545e08b96a9f9b2a51e0ec6fa079b7393b1270544cac4af6f6246fd5","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"a186884d8b4f85878973e1e8bb26766fcd36f0f99234916982d31c2386197c2a","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"a21cc835fa8cdb2ed9033f1ca18d1eb7db85fc60669d3f2f049f9f0f8e99c24d","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"ab6420d788f7161106d32d50e735d0b55e0a7d0b7c8c03f40db9586c7ac5332f","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"b9513cbbafa37244dbcd3207b8b1836ea824ef6ed8a8fa6ee5005da287c245c1","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"bd221a69442a6d79a216e48307db3abafc2724516fd0d3101335d3d4bb683a2d","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"c3ed73d691c5929cdf2946a476c8b001c9536e6031f54f65e36b74ff7e76c90f","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"d798f3f3a8a081a020bf7521957c622331f9901d07425be35f0617500d755d67","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":13.88888888888889,"anchor_occurrences":5,"after_last_push_count":7,"trend_counts":[0,0,0,0,3,3,5],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T21:15:25Z\n Ended: 2026-09-20T21:27:28Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j1718656-svc\", component=\"frontend\", method=\"get\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j1718656-svc).","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T18:17:23Z\n Ended: 2026-09-20T18:33:23Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6867712-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters exceeded 1 second over the past 30 minutes (cluster ci01-j6867712-svc).\nFiring 2:\n State: Resolved\n Started: 2026-09-20T18:17:23Z\n Ended: 2026-09-20T18:32:24Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j6867712-svc\", component=\"frontend\", method=\"delete\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for delete /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j6867712-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T13:27:17Z\n Ended: 2026-09-20T13:41:19Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathLatency\", cluster=\"ci01-j2758528-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} exceeded 1 second over the past 30 minutes (cluster ci01-j2758528-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7035/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101762182411718656","occurred_at":"2026-09-20T19:55:27Z","signature_id":"993e0edbc2802d27b58b7315a4badc5470fe6efc5c6616983981032abe8b49db","pr_number":7035,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7033/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101715422586867712","occurred_at":"2026-09-20T16:49:39Z","signature_id":"a1c099010ae016739013a1e87affb32949a8f1160e4fb45e709b3366b93a0cff","pr_number":7033,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643653972758528","occurred_at":"2026-09-20T12:04:28Z","signature_id":"f483622241db14070571766c5ea34d86fd6f53d96057f1964f32d162917e2451","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101634420027953152","occurred_at":"2026-09-20T11:27:46Z","signature_id":"5b8bff552d46579784f79a0d3b3120eddbccf7ef351d544b8715f930813682e3","pr_number":7026,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","signature_id":"f6030bcc4e4cc5c82d977d09b32388275720da97127703ae213df38d33273f13","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":13.88888888888889,"anchor_occurrences":5,"after_last_push_count":20,"trend_counts":[0,0,8,7,11,6,5],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":37,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:18:44Z\n Ended: 2026-09-21T00:50:46Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4251392-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4251392-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:42:36Z\n Ended: 2026-09-21T00:48:35Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j7913600-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j7913600-svc).","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T21:05:56Z\n Ended: 2026-09-20T21:11:53Z\n Severity: Sev3\n Labels: alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j8645120-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j8645120-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"b10599bdf4bc1308f14fce45589e487e1d0d085cf0ef4c4bb64b2cf8c30f79c9","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","signature_id":"f65d23db3fce211d34d0769e73e3f8f7acba651c0170fe2e1546f79d4a71b579","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"8f4619349028d8271e222f8d6571b2f090f7d7878ecb45ed7f028937641f406f","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"e7c54520064dc8897e885bf4790b5cdcc177d7e87c6cdbf61692ea5647d049d2","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7026/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101600364640866304","occurred_at":"2026-09-20T09:12:27Z","signature_id":"f3bf8d60286004f12d28b5f26936659fa6d1debd6a2443cbc6166170ce25bcb6","pr_number":7026,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":8.333333333333334,"anchor_occurrences":5,"after_last_push_count":12,"trend_counts":[0,0,6,6,4,1,5],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":20,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100fa80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-xdxwcdw5rptk\" to finish creating: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"name\": \"f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.938506094Z\",\n \"endTime\": \"2026-09-21T00:46:21.511326736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:105]: failed to create node pool \"np-1\" for v20261001preview cluster \"v20261001\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00100fa80\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20261001\" in resourcegroup=\"v20261001preview-xdxwcdw5rptk\" to finish creating: GET https://rp.j4251392.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"name\": \"f3174ec5-94e2-47aa-8619-7f3c1d31d5c4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T00:27:13.938506094Z\",\n \"endTime\": \"2026-09-21T00:46:21.511326736Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c6e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-hxgb2rxvk82d\" to finish creating: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"name\": \"2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:48:17.088169395Z\",\n \"endTime\": \"2026-09-20T21:07:20.582868448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:123]: failed to create node pool \"np-1\" for private ingress cluster \"private-ingress\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc00101c6e0\u003e: \n failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-ingress\" in resourcegroup=\"private-ingress-hxgb2rxvk82d\" to finish creating: GET https://rp.j7010688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"name\": \"2ed88cf8-d1af-4b17-9bae-6034a71c6604\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:48:17.088169395Z\",\n \"endTime\": \"2026-09-20T21:07:20.582868448Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000219dc0\u003e: \n failed to create NodePool np-ggj24g: failed waiting for nodepool=\"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\" in resourcegroup=\"rg-candidate-4-20-686sncjvb2sx\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"name\": \"60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:06:27.477664357Z\",\n \"endTime\": \"2026-09-20T20:25:30.377289875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000219dc0\u003e: \n failed to create NodePool np-ggj24g: failed waiting for nodepool=\"np-ggj24g\" for cluster \"cluster-candidate-4-20-ggj24g\" in resourcegroup=\"rg-candidate-4-20-686sncjvb2sx\" to finish creating: GET https://rp.j9550720.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"name\": \"60924801-61a0-4bc9-912c-bcdf013ae1f4\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T20:06:27.477664357Z\",\n \"endTime\": \"2026-09-20T20:25:30.377289875Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"node pool creation did not complete before the deadline\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"297f5e4b1d1f4952837a1cae50b737b42a9bc431965f0ebd876867d9c038a0d0","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101754913297010688","occurred_at":"2026-09-20T19:26:34Z","signature_id":"fec305f359038a8b537a558ba87b3d6d7b2c427d51eb493f15dad5445be91e77","pr_number":7030,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"08debcb95fe76787cc5e7470c732a78966ba147069573bb898c775a88f1cff3e","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"531647f22030c4eb57aa8d0ce0f1192ad57dde45a170d15dbfbc6d84af797341","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"e54b973f5d0dfed8acc1e8e3583260026a6a968d1ae6ee942caf592be98feaeb","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":11.11111111111111,"anchor_occurrences":4,"after_last_push_count":42,"trend_counts":[0,0,30,31,15,4,4],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":84,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":4}],"full_error_samples":["alert fired 24 time(s) (12 unknown, 12 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:36:05Z\n Ended: 2026-09-21T01:14:07Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:37:08Z\n Ended: 2026-09-21T01:15:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T00:37:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"etcd-1\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-21T00:39:06Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"etcd-2\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-21T00:40:06Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-21T00:40:06Z\n Ended: 2026-09-21T01:15:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-21T00:40:06Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-21T00:41:05Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-21T00:41:05Z\n Ended: 2026-09-21T01:18:05Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"etcd-0\", region=\"westus3\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd-0 has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:24:26Z\n Ended: 2026-09-21T00:30:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7913600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2ggt01co6d2ehgcrfpefiv7co64vf-v20260901\", pod=\"router-fbcbc989-n4pcs\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2ggt01co6d2ehgcrfpefiv7co64vf-v20260901/router-fbcbc989-n4pcs has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:24:27Z\n Ended: 2026-09-21T00:30:28Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j7913600-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sv2ggpos1dbpnrhtfim3df37oqp4d6v-q4q2f0t2i0p0l7c\", pod=\"router-cb77f855-wpm9t\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sv2ggpos1dbpnrhtfim3df37oqp4d6v-q4q2f0t2i0p0l7c/router-cb77f855-wpm9t has been in a non-ready state for longer than 5 minutes.","alert fired 20 time(s) (10 unknown, 10 known)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"etcd-2\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd-2 has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T17:59:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"etcd-1\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd-1 has been in a non-ready state for longer than 5 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-20T18:00:47Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-20T18:01:48Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:48Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd-0 has been in a non-ready state for longer than 5 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-20T18:02:57Z\n Ended: 2026-09-20T18:34:49Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"etcd-0\", region=\"centralus\", severity=\"warning\"\n Description: Pod ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd-0 has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"911f13dc127a9ace065557f3d9d6b1c823b667662d3dabe8b6a218f33ca1bf47","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101809250517913600","occurred_at":"2026-09-20T23:02:29Z","signature_id":"7d106244608ace8de6f9cae03dbed9bf2952c22134ad7882a0e61735843e27ac","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"5e756453c54bd40ab2d5dd229f82f628f7919ef41eabbc02f3d959659f4f7234","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","occurred_at":"2026-09-20T14:43:39Z","signature_id":"19e557aaac4e676680853d6de094c41a779d1dca118ed39692cb9878dfd677f7","pr_number":6965,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":11.11111111111111,"anchor_occurrences":4,"after_last_push_count":11,"trend_counts":[0,0,5,1,4,1,4],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":4}],"full_error_samples":["alert fired 24 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:46:08Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:47:08Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T00:47:09Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T00:47:09Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-21T00:47:09Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-21T00:48:11Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-21T00:48:11Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-21T00:48:11Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-21T00:49:07Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:16:10Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:15:08Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-21T00:50:11Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-21T00:51:12Z\n Ended: 2026-09-21T01:18:07Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-21T00:51:13Z\n Ended: 2026-09-21T01:17:13Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-21T00:51:13Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-21T00:55:14Z\n Ended: 2026-09-21T01:14:09Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 2 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:35:55Z\n Ended: 2026-09-21T01:02:51Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T00:35:55Z\n Ended: 2026-09-21T01:06:53Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd has not matched the expected number of replicas for longer than 15 minutes.","alert fired 20 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfama3pu5un8dtplkrue0hb9gd9d-u5c3f9j4q5e2m1g/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustit9omv6o2votoquk6fobdhva9dc-basic-cluster/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-20T18:09:57Z\n Ended: 2026-09-20T18:26:53Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfa5b7n2ii83egk7qcuqspbf8nbe-e4g7e2p0l8n0f4h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-20T18:10:51Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-20T18:10:51Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-20T18:10:51Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susts60dnkbabh5ft9bj4gfhtr5ubsk-x9k7l8s6j9j9a8s/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-20T18:10:52Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustim6s1ajrlhkslrnmndb00d2cdej-a9s9n9i3z7c3h7q/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:48Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustigbiqcsqa7ot259bh7j9f8ldvo4-v20260901/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-20T18:11:50Z\n Ended: 2026-09-20T18:28:49Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susti31lvtu6et9h1g12ocgp7vj6mps-r3a8a9w1u2n1m3w/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2susu70nvmv9oua374lkat5p6me0e8uj-m3e0z8z8q6h4o8y/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustimf10tsndp050c5riq90rci27ia-ingress-cert-le/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-20T18:12:50Z\n Ended: 2026-09-20T18:26:54Z\n Severity: Sev3\n Labels: alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/hcps-j3380352\", namespace=\"ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n Description: StatefulSet ocm-arohcpci01-2sustfejp901oe73sts3aos23r1q49hs-v9h3v9u9m8x4n0h/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"59cd894a17dcad23b6224f622452e17a2a68039a078585dd58b1284a023c6bf0","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"386fa5f6fc68894d00954640db0f0852279a2e3a87df5f417fecec60fd94a6bb","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"8da3d9c87ac6e15f84a876a8ec0eb6d135540c742388bef66277cd5947ec5e3b","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","signature_id":"e66e5d2206cbbff91631193f79b282c5f04f3b6efea313eb1c9fc43af3b6f8ad","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":8.333333333333334,"anchor_occurrences":3,"after_last_push_count":9,"trend_counts":[0,0,6,6,3,2,3],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":20,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":3}],"full_error_samples":["alert fired 24 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T01:01:07Z\n Ended: 2026-09-21T01:19:09Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T01:01:07Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2niah7ifd1s9jttm16f840o7s2bba-c1d1u2p9r6k8c3j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nnkqggaumpd7aknlvveqfl5lapdv-h4h9f6k0o1w3e4w/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-21T01:02:09Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notcp59vbbu2h2jl4hhookui3ni8-ephemeral-disk/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-21T01:03:05Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2no5m2g2a1qcfintm8jnt5d7r4pui-y4s4v4u1n8i5y3g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2njptuelc95q6e2m2dhovv3otu6t3-v20260901/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:16:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nicbl3b9roljek1dt21vuafpgd0h-private-ingress/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:17:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 17:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nngnkkmkc9hqb21a6e6r19ffc9f3-a6p1z2m3j7z8r2k/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 18:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nndocsvsbeib49ejqtshhqkhspgc-v20261001/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 19:\n State: Resolved\n Started: 2026-09-21T01:04:08Z\n Ended: 2026-09-21T01:17:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2notvju96brqeq57d8bslufo78773-c2z6q7g5n5d4f3l/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 20:\n State: Resolved\n Started: 2026-09-21T01:05:07Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 21:\n State: Resolved\n Started: 2026-09-21T01:05:07Z\n Ended: 2026-09-21T01:17:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2noq6s2bv2p71j6282l96pjjrnpmp-k5g1v9y7w5b4u1n/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 22:\n State: Resolved\n Started: 2026-09-21T01:05:07Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2os5r8rsidk6oq66pn7nilat6pj6k-agg-cluster/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 23:\n State: Resolved\n Started: 2026-09-21T01:06:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 24:\n State: Resolved\n Started: 2026-09-21T01:06:08Z\n Ended: 2026-09-21T01:19:08Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/hcps-j3696384\", namespace=\"ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2nkute35u0qtm816oest3s71st9q6-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:49:52Z\n Ended: 2026-09-21T01:04:52Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-5tv64\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T15:26:04Z\n Ended: 2026-09-20T15:41:07Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6358528-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.107:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6358528/providers/microsoft.monitor/accounts/hcps-j6358528\", namespace=\"ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-x5q5k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment ocm-arohcpci01-2suq8o78o0jbc33n4l57r98vk1s6tm51-ingress-cert-le/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"a34c4272b290720aafd3b29a745fe60b6976f6c8d6360946f5a75edf1096c9ec","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"a2c1fbf6e64cddf7a46eaafcc044b53f14f98a5940a39f4dbc58486310e950ce","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101673315566358528","occurred_at":"2026-09-20T14:02:20Z","signature_id":"84036f94843f01e49f13138f55d77ae74e49b96060ac2cc402fa01f03e462128","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":8.333333333333334,"anchor_occurrences":3,"after_last_push_count":7,"trend_counts":[0,0,4,2,1,3,3],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":13,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":3}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-21T01:28:40Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4251392-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5hkmr\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-5hkmr (namespace arobit) on cluster ci01-j4251392-svc has averaged 126.7% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-20T18:55:31Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j3380352-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-qgqnm\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-qgqnm (namespace arobit) on cluster ci01-j3380352-svc has averaged 126.9% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n State: Fired (not resolved)\n Started: 2026-09-20T16:39:02Z\n Severity: Sev3\n Labels: alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j9006208-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vbl5w\", severity=\"warning\", team=\"hcp-sl\"\n Description: Container fluentbit in pod arobit-forwarder-vbl5w (namespace arobit) on cluster ci01-j9006208-svc has averaged 126.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"39e21beab76cee821e0ed80a6601d86d04d54351b87f382522d67356ee5fa584","pr_number":7036,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"b4c5714662b6eebf9aeab7aa2383a89d5f76557fa6ae1c0f2264a9d31c883f89","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101683713959006208","occurred_at":"2026-09-20T14:43:39Z","signature_id":"fb812e0e230afdfaf46a5827aae975ba58d618c82618d4f8ed79f5468297e6a0","pr_number":6965,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.555555555555555,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,2,2,1,0,2],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:59:00Z\n Ended: 2026-09-21T01:07:59Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2niah7ifd1s9jttm16f840o7s2bba\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2niah7ifd1s9jttm16f840o7s2bba/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-21T01:01:00Z\n Ended: 2026-09-21T01:07:05Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-21T01:02:03Z\n Ended: 2026-09-21T01:09:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-21T01:02:03Z\n Ended: 2026-09-21T01:09:02Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3696384-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.21:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3696384/providers/microsoft.monitor/accounts/services-j3696384\", namespace=\"klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-m2cgj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sv2notcp59vbbu2h2jl4hhookui3ni8/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.","alert fired 16 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfama3pu5un8dtplkrue0hb9gd9d/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n State: Resolved\n Started: 2026-09-20T18:23:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustfa5b7n2ii83egk7qcuqspbf8nbe/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n State: Resolved\n Started: 2026-09-20T18:24:36Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustit9omv6o2votoquk6fobdhva9dc\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustit9omv6o2votoquk6fobdhva9dc/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n State: Resolved\n Started: 2026-09-20T18:24:37Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2sustim6s1ajrlhkslrnmndb00d2cdej/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n State: Resolved\n Started: 2026-09-20T18:24:37Z\n Ended: 2026-09-20T18:30:36Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2susts60dnkbabh5ft9bj4gfhtr5ubsk\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2susts60dnkbabh5ft9bj4gfhtr5ubsk/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n State: Resolved\n Started: 2026-09-20T18:26:36Z\n Ended: 2026-09-20T18:32:37Z\n Severity: Sev3\n Labels: alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3380352-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.106:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3380352/providers/microsoft.monitor/accounts/services-j3380352\", namespace=\"klusterlet-2susu70nvmv9oua374lkat5p6me0e8uj\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-9wqft\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n Description: Deployment klusterlet-2susu70nvmv9oua374lkat5p6me0e8uj/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"5c8b752be3d2e2575305bb4467922ed388fb9c7274d746c67ce437e73fc6372a","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"8389beb2f2127c28d1c407bb2631de05104a4b0dfdbf0a89c145c62d14049937","pr_number":6965,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":5.555555555555555,"anchor_occurrences":2,"after_last_push_count":5,"trend_counts":[0,0,3,0,2,0,2],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0027fddb0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.001s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc0027fddb0\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00280b770\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:383]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n \u003c*errors.errorString | 0xc00280b770\u003e: \n certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101643184168767488","occurred_at":"2026-09-20T12:02:36Z","signature_id":"0899e170df3dbff91a3d93d67607d26e9d9e1d55e607106e4712f92b087c03fd","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101575058240573440","occurred_at":"2026-09-20T07:31:53Z","signature_id":"73ffdde7c35ef7b4f90a0d9d0266878e677b3024d0af73c677985cd106c56138","pr_number":7025,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"72716b475dfdb36ca6de5b4a7866bbc958e1921fb2c6ea0b3467c231190cd12c","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-wg24nd-tjd5hx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-wg24nd-tjd5hx\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-np-version-upgrade-cluster-wg24nd-tjd5hx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:136]: failed to create cluster customer resources\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000f86780\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-np-version-upgrade-cluster-wg24nd-tjd5hx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-np-version-upgrade-cluster-wg24nd-tjd5hx\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-np-version-upgrade-cluster-wg24nd-tjd5hx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8d220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-cc6rq5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-cc6rq5\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-pullsecret-hcp-cluster-cc6rq5' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:104]: failed to create customer resources for pull secret cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000a8d220\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-pullsecret-hcp-cluster-cc6rq5\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-pullsecret-hcp-cluster-cc6rq5\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-pullsecret-hcp-cluster-cc6rq5' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","signature_id":"7315612725ec25e205b11f3adbd85547f42e9e5cff4ae5b4c955333bbd4ccd4f","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","signature_id":"8a93b4c61b0436ed275798352e60b97494e315997f9735dcbbf3fcea1cb8b1bc","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"44fc5c170ee5969da686dbaeb61a39c91b4e9f01cf7a4324b16151d7f8a7811a","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider, ignition-server, ignition-server-proxy]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c0f60\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2fppphk96nck\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"name\": \"648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:38.502314717Z\",\n \"endTime\": \"2026-09-20T18:19:40.439163186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:88]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0005c0f60\u003e: \n failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-2fppphk96nck\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"name\": \"648b059b-8450-4d5d-bb2f-41ba8931b7e5\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:38.502314717Z\",\n \"endTime\": \"2026-09-20T18:19:40.439163186Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000696760\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lzkpz5ftlkll\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\",\n \"name\": \"83245618-9845-4872-97a3-4ee524027d39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.101717721Z\",\n \"endTime\": \"2026-09-20T18:19:10.391207338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:107]: failed to create seed HCP cluster basic-cluster\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000696760\u003e: \n failed to create HCP cluster basic-cluster: failed waiting for cluster=\"basic-cluster\" in resourcegroup=\"rg-cluster-nsg-subnet-reuse-lzkpz5ftlkll\" to finish creating: GET https://rp.j3380352.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/83245618-9845-4872-97a3-4ee524027d39\",\n \"name\": \"83245618-9845-4872-97a3-4ee524027d39\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-20T17:50:02.101717721Z\",\n \"endTime\": \"2026-09-20T18:19:10.391207338Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider deployment has 1 unavailable replicas, ignition-server deployment has 1 unavailable replicas, ignition-server-proxy deployment has 1 unavailable replicas]\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"e6538054ab0559cddb2c0ed63235c48a4d5cd57bef25e042c4c66fb7e93ba92e","pr_number":6965,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101720599553380352","occurred_at":"2026-09-20T17:10:13Z","signature_id":"f37d1bd86a2dec4c46be9ce8fb4c03ed08425d29a0d32e20dd962b464a680228","pr_number":6965,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b8593d4d91aeed30f93b501cb1ed65e6103f69a14e5ea5e4f7e26f7c2c0042f1","failure_pattern":"expected node pool nodes to report at least one release image ref before upgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":2,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:180]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:474]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:474]: expected node pool nodes to report at least one release image ref before upgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","pr_number":7032,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"51a09df5028813b02368114a6b230a29919a67728e7a9fb6c2e76b59211f51e4","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef5080\u003e: \n DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n \u003c*fmt.wrapError | 0xc000ef5080\u003e: \n DNS for route host agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-sfwxt.apps.aro.v20260901.iay7.j8645120.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6965/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101763320288645120","occurred_at":"2026-09-20T19:59:59Z","signature_id":"571f2e2e0cd4941242495f0f83848c90e92c491345357a2fc6f06f0266aa0a09","pr_number":6965,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"e07cda02c84cd6ea2552fbcc5179ebfd519e66665b62547f1ea88bc8136bc936","failure_pattern":"ERROR CODE: DeploymentFailed; detail code NotFound; detail message Resource '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.; provider Microsoft.ManagedIdentity","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-20T17:31:48.163Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n Kind: ARM\\n Template: templates/mgmt-cluster.bicep\\n Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-20T17:31:48.670Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-20T17:31:49.380Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1872640-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6\ntime=2026-09-20T17:33:49.754Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.Resources/deployments/113c13d4bee9395d4112e94ba8bdfafc6cef483f6067a14e96151dee740150d6/operationStatuses/08584116813764412291\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentFailed\\\",\\n \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n \\\"details\\\": [\\n {\\n \\\"code\\\": \\\"Conflict\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.Resources/deployments/aks-cluster-post\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n \\\\\\\"details\\\\\\\": [\\\\r\\\\n {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n ]\\\\r\\\\n }\\\\r\\\\n}\\\"\\n },\\n {\\n \\\"code\\\": \\\"NotFound\\\",\\n \\\"message\\\": \\\"{\\\\r\\\\n \\\\\\\"error\\\\\\\": {\\\\r\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"NotFound\\\\\\\",\\\\r\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"Resource '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1872640-mgmt-2/providers/Microsoft.ManagedIdentity/userAssignedIdentities/prometheus' was not found.\\\\\\\"\\\\r\\\\n }\\\\r\\\\n}\\\"\\n }\\n ]\\n }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101718617941872640","occurred_at":"2026-09-20T17:02:21Z","signature_id":"cc1dc5d9930eb4720cd4205037d7349da3ed760315a1e40606e0aebfe552ae57","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"badae18d6ca0a210966f4d55e11bcbd9bee452e9a42e4ebbf3c4a1925b53c93d","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,2,0,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001366090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-fxwzs2\" in resourcegroup=\"rg-np-downgrade-fxwzs2-xgl2x967w267\" to finish deleting: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"name\": \"bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:01:37.424253631Z\",\n \"endTime\": \"2026-09-21T01:25:46.248853963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2nnff2dqdtbc8d6g85073gq2fnh75 still exists (deletion dispatched at 2026-09-21T01:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n \u003c*errors.joinError | 0xc001366090\u003e: \n failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"np-downgrade-fxwzs2\" in resourcegroup=\"rg-np-downgrade-fxwzs2-xgl2x967w267\" to finish deleting: GET https://rp.j3696384.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"name\": \"bf94f468-3ed1-44cc-a51b-69992a140dfb\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-09-21T01:01:37.424253631Z\",\n \"endTime\": \"2026-09-21T01:25:46.248853963Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sv2nnff2dqdtbc8d6g85073gq2fnh75 still exists (deletion dispatched at 2026-09-21T01:01:38Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101819340423696384","occurred_at":"2026-09-20T23:42:35Z","signature_id":"e8bc9a8486674bdc1a81049ad931ccbb5a9f98bfa53795eca763544cc96cde0e","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001f2540\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-g8jlwfczq87l\" to finish updating: GET https://rp.j7871744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0001f2540\u003e: \n failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-g8jlwfczq87l\" to finish updating: GET https://rp.j7871744.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: ResourceNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"ResourceNotFound\",\n \"message\": \"The resource 'locations/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039' under resource group '' was not found.\",\n \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bc0c4f76-9396-400e-ad88-33ac328d5039\"\n }\n }\n --------------------------------------------------------------------------------\n \n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7019/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101522358387871744","occurred_at":"2026-09-20T04:02:00Z","signature_id":"07b9cc47f1337561bab912a04c50994e47b493cb94f69944313b85e20e5e8827","pr_number":7019,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\nUnexpected error:\n \u003c*errors.StatusError | 0xc000ac2640\u003e: \n Operation cannot be fulfilled on nodes \"k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\nUnexpected error:\n \u003c*errors.StatusError | 0xc000ac2640\u003e: \n Operation cannot be fulfilled on nodes \"k6x3v2v7z6d4m6q-cordon-np-7ph5q-cjmjj\": the object has been modified; please apply your changes to the latest version and try again\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101610585002086400","occurred_at":"2026-09-20T09:53:04Z","signature_id":"39cf09c4d4c3e066082fd6d0301325efef03f8489b16e7088e47c2ad9b88bbaa","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f570ff2ef05014779bb7489dca295e256a69aead33d6bff8319ed54db32fd9e1","failure_pattern":"UpdateHCPCluster20260901 timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ca8a20\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kms_key_rotation.go:184]: failed to update cluster with new KMS key\nUnexpected error:\n \u003c*fmt.wrapErrors | 0xc000ca8a20\u003e: \n failed waiting for hcpCluster=\"kms-key-rotate-422\" in resourcegroup=\"kms-key-rotate-w7nhqllrvzc4\" to finish updating, caused by: timeout '23.000000' minutes exceeded during UpdateHCPCluster20260901 for cluster kms-key-rotate-422 in resource group kms-key-rotate-w7nhqllrvzc4, error: context deadline exceeded\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7025/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101647291294158848","occurred_at":"2026-09-20T12:18:55Z","signature_id":"e0ec91f3cf6a62bae5b4c3d56509ed71730ec1128a9898d0313dd55a382d4fa9","pr_number":7025,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"55fc315450b35ec48de3526d899619e34f6aae1d138260064fe2d53d5403f86e","failure_pattern":"VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet \u003cnamespace\u003e/global-pull-secret-syncer has no desired pods scheduled","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010007a0\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_pullsecret.go:183]: failed to wait for global-pull-secret-syncer DaemonSet to be ready on nodepool nodes\nUnexpected error:\n \u003c*fmt.wrapError | 0xc0010007a0\u003e: \n VerifyDaemonSetReady(kube-system/global-pull-secret-syncer) timed out after 10m0.006s: DaemonSet kube-system/global-pull-secret-syncer has no desired pods scheduled\n ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"f61d48f7e4dfc0d5ed670d4815de996c6e11f3ee6013821737755d17d338fd53","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,2,1,3,1,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:38:51Z\n Ended: 2026-09-21T01:06:51Z\n Severity: Sev3\n Labels: alertname=\"KubePodCrashLooping\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.115:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4251392/providers/microsoft.monitor/accounts/hcps-j4251392\", namespace=\"ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"4d11d27a-526b-40d4-812b-9a43bae7deb5\"\n Description: Pod ocm-arohcpci01-2sv2ibgc88ck9ggpl6jfg4j7vh3t3p5h-u3g5z3d7e7t7o3d/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"14896a8781a6fade79c483f0daff72f9c39c1157a6161055b98515a6ef455b60","pr_number":7036,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,3,1,4,1,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":10,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-20T17:13:02Z\n Ended: 2026-09-20T17:29:03Z\n Severity: Sev3\n Labels: alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j5465216-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7030/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101705684675465216","occurred_at":"2026-09-20T16:10:57Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7030,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,5,1,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n State: Resolved\n Started: 2026-09-21T00:30:39Z\n Ended: 2026-09-21T00:36:37Z\n Severity: Sev3\n Labels: alertname=\"KubePodNotReady\", cluster=\"ci01-j4251392-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-5q82q\", region=\"canadacentral\", severity=\"warning\"\n Description: Pod arobit/arobit-forwarder-5q82q has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7036/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101812104104251392","occurred_at":"2026-09-20T23:13:50Z","signature_id":"258bf6b8e2e6a15e2ce38412cd2942fddf00da1a19bafe63316ac35a44ba6d7c","pr_number":7036,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"422ddc32f327db7a61c881b4411ef7767ec79e98b46476bd6a0cffdccf708740","failure_pattern":"expected all nodes to have label 'key1=value1'","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n \u003cbool\u003e: ...\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:158]: expected all nodes to have label 'key1=value1'\nExpected\n \u003cbool\u003e: ...\nto be true"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"48929c5f392dda19c39d764a9d4117d26ea3b70f835a97462a0249c92cc9fdf4","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"97763ea227d764b6cace3c59c3e2291599eb63cdd34dd1463aad690bc17db216","failure_pattern":"expected node pool nodes to report at least one release image ref before downgrade","search_query":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:753]: expected node pool nodes to report at least one release image ref before downgrade\nExpected\n \u003cset.Set[string] | len:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"31f7686d857db3ec2fac93987fae6b6a3245782f4b94f1b84b4b75f60d2d2c13","pr_number":7032,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"990ae447fc69c4a73fb4f72088b1a7f653aec54482f588061e215576813bf077","failure_pattern":"node pool \"\u003cnodepool\u003e\" should have at least 2 nodes (autoscaling min), found 0","search_query":"node pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":2.7777777777777777,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-14..2026-09-20 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:518]: Timed out after 900.001s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:505 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: ...\nto be \u003e=\n \u003cint\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:518]: Timed out after 900.001s.\nnode labels/taints should appear on the intended node pools of cluster \"agg-cluster\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:505 with:\nnode pool \"agg-np-a\" should have at least 2 nodes (autoscaling min), found 0\nExpected\n \u003cint\u003e: ...\nto be \u003e=\n \u003cint\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7032/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2101748651339550720","occurred_at":"2026-09-20T19:01:41Z","signature_id":"204e1750cfbcdccc7a1aa87cf8b29089bd7645257cbc1509ff43f3d1c64bf5f6","pr_number":7032,"after_last_push_of_merged_pr":true}]}]}]}