{"meta":{"start_date":"2026-09-17","end_date":"2026-09-17","start_at":"2026-09-17T00:00:00Z","end_at":"2026-09-18T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T12:04:27Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":196,"failed_runs":176,"raw_occurrences":366,"matched_occurrences":271,"runs_affected":81},"rows":[{"environment":"dev","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":97,"runs_affected":38,"failed_runs":38,"run_impact_percent":19.387755102040817,"anchor_occurrences":97,"after_last_push_count":49,"trend_counts":[0,0,0,0,0,71,97],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":65,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":6},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":5},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":8},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":4},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":7},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":5}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f703e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-lxp82nsl4llc\" to finish creating: GET https://rp.j9381376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9411175-a2f7-4b2d-9324-66fd2acc379f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9411175-a2f7-4b2d-9324-66fd2acc379f\",\n      \"name\": \"f9411175-a2f7-4b2d-9324-66fd2acc379f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:50:07.442410011Z\",\n      \"endTime\": \"2026-09-18T00:19:13.547086739Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:87]: failed to create HCP cluster \"fips-enabled-cluster\" with cryptoRestrictions set to FIPS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f703e0\u003e: \n    failed waiting for cluster=\"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-lxp82nsl4llc\" to finish creating: GET https://rp.j9381376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9411175-a2f7-4b2d-9324-66fd2acc379f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f9411175-a2f7-4b2d-9324-66fd2acc379f\",\n      \"name\": \"f9411175-a2f7-4b2d-9324-66fd2acc379f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:50:07.442410011Z\",\n      \"endTime\": \"2026-09-18T00:19:13.547086739Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m11s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007926a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vsg6jbtlvgqx\" to finish creating: GET https://rp.j5944704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\",\n      \"name\": \"ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:20:00.021017752Z\",\n      \"endTime\": \"2026-09-17T23:49:08.836539572Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:106]: failed to create HCP cluster \"cilium-cluster\" with no CNI and private etcd\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007926a0\u003e: \n    failed waiting for cluster=\"cilium-cluster\" in resourcegroup=\"complex-cilium-kv-vsg6jbtlvgqx\" to finish creating: GET https://rp.j5944704.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\",\n      \"name\": \"ced041bf-9cd3-4dc0-8816-ef44d4b87c6b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:20:00.021017752Z\",\n      \"endTime\": \"2026-09-17T23:49:08.836539572Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 23m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe8980\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hgg4wl6sg8sc\" to finish creating: GET https://rp.j0002944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f65c2fa-1e9a-4309-96ea-66a270395b00\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f65c2fa-1e9a-4309-96ea-66a270395b00\",\n      \"name\": \"2f65c2fa-1e9a-4309-96ea-66a270395b00\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:30:18.236986606Z\",\n      \"endTime\": \"2026-09-17T23:59:22.774158112Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fe8980\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-hgg4wl6sg8sc\" to finish creating: GET https://rp.j0002944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f65c2fa-1e9a-4309-96ea-66a270395b00\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2f65c2fa-1e9a-4309-96ea-66a270395b00\",\n      \"name\": \"2f65c2fa-1e9a-4309-96ea-66a270395b00\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:30:18.236986606Z\",\n      \"endTime\": \"2026-09-17T23:59:22.774158112Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.33 is Partial (want Completed), started 24m14s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721105009381376","occurred_at":"2026-09-17T22:58:35Z","signature_id":"d601eba3abc4f82457501b40b90cc5d7561cc3383a0ad74d5298ef8e79a87795","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100714303135944704","occurred_at":"2026-09-17T22:31:33Z","signature_id":"7468e248b9c77a11eebc376f236b7610c67fe482e15792bf83562366e5934d5f","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713215490002944","occurred_at":"2026-09-17T22:27:14Z","signature_id":"b9f0cd575efa9d87456c559803742a6d89fb91600ce45815cecb4689bcc702f8","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100713215490002944","occurred_at":"2026-09-17T22:27:14Z","signature_id":"db652501caa7a4389b880b5afa48aab924fa2baf6e02836e6489573d8b826dbb","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"6dff3d6d762cfe52b458282a2d61a5824494a4b7fbf9f3aab1ba2e26d05698c5","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"7bfd2cf527b8d88bab96af9ab4da562292d265e26e2314dcdcf6bffc6b100ac5","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"d74cdb3a0399ea27cc6c2ee2e3e86b457bc6ff248822afc15a1f630792589716","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"e593b0bc62b406e1f219043a4233fe6d4b2427734c6df50023d50f3222509f40","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"0a1d46d442635db3b7061c82189b8035af60a48310a53ecc646e31620910a1cb","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"50d86109b369a86d21ce7f771945b06926a4c38d2d49433ad0d362386d5132f7","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"78fecc2305e80f61dc9910998cc7c318760f78b41aed63eb7f46b3ce837e0d9f","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"aef26cedd47d167588772783661deaa26628c5b57bcb923a1a0a520ea749663a","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"b07610b43263bb7e953aa2613bf43c66ef66f67a6d400930f3acc7d9b2871dfc","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685757428535296","occurred_at":"2026-09-17T20:38:07Z","signature_id":"f8f97e2decd1a3d48fb001cc1c33150efca6eca10d42b34bfc0118d3f5dd19c8","pr_number":6998,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","signature_id":"68698c4e2d81e96a811614a8eb209ad22dc4f07b9b07bee4f975ea46939d038d","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","signature_id":"62a250c534c20b5ad145d249c89f89739cbae0c9b5704a8277cba737ff1f0047","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","signature_id":"824964c0c575d45ef044ea907156d5bdd2f98d69978b43f2f1fb3b622414875f","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569791986143232","occurred_at":"2026-09-17T12:57:19Z","signature_id":"f24c215311a297177eaa94e5511fcc51ba57d864ab61313b4419552b1cf74930","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","signature_id":"903f3f622deb52a1f1d8dea7e418d3ea7d8db5819b70eccb79ae8ea0a5299dd4","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"1fb4041dc59d81c7e183948830ae7ac1b2783b718bf3d7631b04090e55e3f847","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"3a922f4d19da47f001e6311e71c0f1a6d255911648780e0fdc7768301e19e9b8","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"e828cec50a4c240181974579c6679994f89f3d798f5316edd2d635df29854690","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","signature_id":"9771d1687cb82ab4b14c0f80173555cf109826201233928e90423e0171bc2724","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"85532b20bf57446a7d41586b4163ec6f19b5563d86b6207d2b7d5ab6951e2651","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"ab6e130ec463d100706975248c724bd66e5d7d3642c918a3434eeabaf4b5aa60","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"b8f768f8fc135387ca0e0f20dc96f1cad4999bade1e76fff9ca2066634cf47d3","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"cb6c78925476f970ec41ca2cc86883f3fe25eedebef129a57d35868eedaf968c","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"ff4a343919b36fa3670725111608ea3b35a12089dab63a806e3c5e121a1fbb9a","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"236cb0e5207b1956ffac77c17fe63e39c6df7e6a37c02c56004d161df8a008ca","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"39a1e24d4800633f0d0b42d4f18189154efcf362e824edcdb07cb8b8b0dd68d9","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"401c9dc75b4b5ec84bc7eacc4dd5a3649404e5b9bb0cad164dff2ac11981d6c4","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"4dab39d1b1d724fed25dc69d84d189cab2ac6a8152b77e555635e3621d133e55","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"bb7a8e5fa783928e378dcc846c79e38f12b390e8f34e30b5e6460dbc694c3af8","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"ebcbd7286e42fab84ff7ea3f9b915f4d4683f1309800106ed6220b22cd9d54e2","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6994/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100557383175507968","occurred_at":"2026-09-17T12:07:52Z","signature_id":"ee498f50c2b2ac982c301f4ac0cae3713723036b9fbee8e63fbb794dec859320","pr_number":6994,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"d54234423202acb80a8cc7b79b53caff0b960878b2770d3f2b50c496f32d7589","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"f4470bf38da6c976133b96af81d9ecd329780f2aa1f6bfa931cdb8bcafadf34f","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"8063e53d23cc71e09e9af725e9b15fa0500e6877a011fce15107ebe822270942","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"8cd2901e21933a26dee207d65687d26f76901c588c4dc077ff1cf8179eac46e9","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"c31f9f905858e3bef622b1331144e2d854b98bc4c9aa6c6098d97c66fd0a17ca","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","signature_id":"9577b3508cc1b362f70a7592604f657ac43e72478f7b4c69c3b22190b77112b8","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"2cb0fa49d019c2b0a11abf2d3312cdc656ba3974fcf4c96aa2f18765ffe8dee0","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"61b54a22bdc6f8db1004cb461b9c1ea0ce8e7bb226818ce98016843218649c01","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"6989180271f1d05831c6e509c6cc427effaad2b85f3bbba286006443200fcb35","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"78cd64b525be75a4d9388fde337bd307c022a2d8ce21d0ffc33bf9360fed27a3","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"7cf6bbc9da68baad7e4462e2758e70a60f2061703be4a96f8198d3c3759e5683","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"969e77e201e0ea5c3383048290794e2481a2bc4e0d76aac6da2b54ff4ff6694c","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"a21f245f7a5f77e335623baa9b0ddbede34c06b521c1c7c4d490e127ce4037e7","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"cd184104708d45339d6bffb3c0990b1efb5eafdada1ae446f6ffcf492fc8f488","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"f0d309a90401bc7c464e4f86124fff842aa1f093303fd8bfa28b66ac92405fe2","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"fe2081edacd08521e3b4731165e95ed1b4f17a242e9810e3d19e5b07192590fb","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100537768756121600","occurred_at":"2026-09-17T10:50:04Z","signature_id":"0d5854936e81c77013558f96a12c534160fb9ce058180d02b89c4d9a3544856d","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","signature_id":"3596b17636b3dd9dc619bcf3bc7003efbdc9033c334b506d05c614ad30a25383","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","signature_id":"b8057328bdd22a04aeb32a86271770e8c39d4d2cd6eac75a438bd766471d440b","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"8bdd74c61a93beb235e7d38100063c0ca3d930717abd8601b9cb4dac72149e51","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"d6f32f2cee9bee5817a4bb9c6f37367a9340014362f5ecc9c490646d7eab987e","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"7cee4e5352bcdeee9af619ecd98980da8f5a54689aadb5ebc237dc1aa40f7036","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"837607adb29247003f557d4cb6ffdfdb8aeac843b72cf365de5b9d8cdce185b2","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"b097d1bd6e31b2848274482e1d375642d3af07f55f2c3adb043f6e076a9933e1","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"b662a9003e08c3ef5e2286107d4a3536bcba752aaf78844948d28b110eee221d","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100520057816748032","occurred_at":"2026-09-17T09:39:42Z","signature_id":"2cc7ca1e205a8dfaf045d2763125c668a7279c4261f08b6ddee57e833c36210a","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"2cf84524e6ccdd0a761441daa39b744a2ba3952626e9461b7d45c15b3bcfd754","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"3f28cae00bcc4e17acb25bb050d65aad4ea199a510b36c178d2d712889664cab","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"acc3649bd8151c983690e39ec8f25e1c5473a348f8d0898d6d79bd8563a49a87","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"dc4d8b8d1deffa568a3e69aa51d30ab552205c7e9c9f140c68e72f873ccbd501","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"2ac60a748539f98f2c88eefc3d7d9d565740acd60ee833fc9852acdd1cfcf35e","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"6be75bbc95c50cbac96f78528dfa07add64f7a96b8fe35fe1510e1615554ba67","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"7f2b1821d50e299f9faaaf1ec8623b52220c8b011f2741480e4d05563a5a26c3","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"706ec129f8f00b0fced3839fa11ea395190e7f746c1a75084493342ffbc32a92","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"1254becb25675dbd9ede114df6e32fd10e76cd84f176a807e3d5dc72b936cfbc","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"1f2d8a520d63c686030e3b1f47535fed1c6501ab4abdd19d36fb0c76229a1cb8","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"32d4bd52ce4e4bf4bbb0c374a620144f8370e122a59cb1cce2ba39c24a22fcac","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"71eb266b069aadc565cf2dfa5c38d0b00bec544a93f9accc8e29672f14a64cd5","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"833379ae7b454d610a7efbf91b7df6a563effdab3e7cab0f6ff3de3a51cd9ae1","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"88399b6110d57a9572b0144a071be4bf2ca4aeadc0f450fbc20ec223beef26bc","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"ced0fb0de9a8798a8f89849b4e849e1ad6b42de9776ac8d0c57c01c8648db002","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"ffd6f4632b90a496bd463d4995c3c420f976f2007427581611b1c7bcf4dd4b9b","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","signature_id":"109feb9a36dbfe6e2dc23c6d08e79142806f38590e9a903022f2d16ea982a69f","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","signature_id":"38588499aa79b332fce0da4da61a0607d4dd3683eb3c9cf92c9d624bfee5f925","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"d26e879298c7f737b9ae16dbe160a711548fa3f84ab3c2a0048bc582a13e4076","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","signature_id":"008a91fd3e3e5f5195f74645a3769624ed2be2ab43fcea742cd06f90df27599f","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","signature_id":"6b862675f286ba82e4aaaeabbdde3ab8b1accc783182f06adb944ac909b9f88b","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","signature_id":"82d6e28574c7f34a07c333ca94f997752b96649f6b80a188942914d02804a6b8","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","signature_id":"2938ba362ea03badfa4fef3a5a8b455d8db3137be286b0a32f0e06cb64bc56e1","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","signature_id":"89c35af4f14189b65c64576cd7490a0088f1c8ab485b614bf43cd6ed4c19e683","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","signature_id":"72e9dcaaa03f26fa26de2141ffb6133bfeb9f62e98e7bbfc149c580a12c625ab","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100404999925272576","occurred_at":"2026-09-17T02:02:30Z","signature_id":"17d592d037b2f2a804c6494b323dd9c8826c7ee66667ea66a41a04ee3ac208fe","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"1c3549dc55bc6c387adb5760f50c6e243c8a43f7b39ed0bf467dc5a0511968bd","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"3287e14a7b0a164b12f072d969ca8183f1647cf1f1f00a66de841c81d64ef6e4","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"a5eed4f538e16bf946e6c1d7bf29b73fc650cf9bfa21cbd95eec591f73113f97","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"0d64bfff34a7c1f61cc366edc7ee533840e5653ed9071035a92b925eab037b24","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"2913423376b3c9240235607ee0d4c675506def5b909b1312a4bdad97516cd24d","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"428648ff0c0135e57b228a05c39d11591ece01ed750e930d86d83fe847be740f","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"49af2dca8e5069846af20cde9f9397824ce26e8ea2d79c52e7c7b30e8faed262","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"721faaab8ec90430d24300487ededdb0a93721318cf4132ab4ace8fde5518506","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"a2ee109ec25c2b87338aedcebe4c8cc584f5a143bdc97ee69a387447d2c549b6","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"21b15360431929bdb593a2107c3896a1ca85c63523d6f4c20410fb74f623d0a9","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":31,"runs_affected":31,"failed_runs":31,"run_impact_percent":15.816326530612244,"anchor_occurrences":31,"after_last_push_count":24,"trend_counts":[0,0,0,0,0,30,31],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":61,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":31}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-18T00:02:16Z\n  Ended:       2026-09-18T00:27:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j9381376-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st2tc78t6cvdchmjv242i9bdc8rpm9q-k2h4w6t5x7l3o9z\", pod=\"router-586c95bb7f-lzcz4\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2st2tc78t6cvdchmjv242i9bdc8rpm9q-k2h4w6t5x7l3o9z/router-586c95bb7f-lzcz4 has been in a non-ready state for longer than 5 minutes.","alert fired 9 time(s) (3 unknown, 6 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T23:40:23Z\n  Ended:       2026-09-18T00:06:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4009472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st2jdcitjtn6r1jl4q34p61lesqvar0-l2u3i7e5p7i0m7j\", pod=\"router-dd88b46c7-ckx8f\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2st2jdcitjtn6r1jl4q34p61lesqvar0-l2u3i7e5p7i0m7j/router-dd88b46c7-ckx8f has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T23:42:25Z\n  Ended:       2026-09-18T00:11:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4009472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st2ji98etmkm3fuvovie18d64b0rt6v-v20260901\", pod=\"router-7fd9f9b4c6-gn2vk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2st2ji98etmkm3fuvovie18d64b0rt6v-v20260901/router-7fd9f9b4c6-gn2vk has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T23:42:25Z\n  Ended:       2026-09-18T00:06:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4009472-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st2ji98etmkm3fuvovie18d64b0rt6v-v20260901\", pod=\"router-7fd9f9b4c6-q4q5l\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2st2ji98etmkm3fuvovie18d64b0rt6v-v20260901/router-7fd9f9b4c6-q4q5l has been in a non-ready state for longer than 5 minutes.","alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T22:37:27Z\n  Ended:       2026-09-17T22:43:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5842688-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st1lu1gnkusrr9aaeo128agqit9e6jo-a8y8c4i0m5j1k4s\", pod=\"router-6f4d998d46-d2xxt\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2st1lu1gnkusrr9aaeo128agqit9e6jo-a8y8c4i0m5j1k4s/router-6f4d998d46-d2xxt has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T22:37:27Z\n  Ended:       2026-09-17T22:43:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j5842688-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2st1lcku7i9kn16epu26j9jmha4t9219-t5q0q7d1s0b0b2h\", pod=\"router-5b9f46c564-vq9rt\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2st1lcku7i9kn16epu26j9jmha4t9219-t5q0q7d1s0b0b2h/router-5b9f46c564-vq9rt has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721105009381376","occurred_at":"2026-09-17T22:58:35Z","signature_id":"82e0a12c7cd95fd907f63543eeac97bdeae8d5f9e5a424bdde230db4ef5d4c57","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"1033d58df80ee9a7d767bdc19425ad417e226db41f4f4ad1a7aa6ed4d2688d8a","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7002/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100699515785842688","occurred_at":"2026-09-17T21:32:45Z","signature_id":"18121697c147e2ac50157b93c7f4175f042996dac56448b883bc040150ef3bb5","pr_number":7002,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6968/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100697739275800576","occurred_at":"2026-09-17T21:25:44Z","signature_id":"d2aa839fc01483bc5381babfe51639f5c2325311fddfd8c5d39caa32de46dc09","pr_number":6968,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6998/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685757428535296","occurred_at":"2026-09-17T20:38:07Z","signature_id":"927ffe3fc5118bce586a8a07b93ed80685d7fe4d247724a208069edabb849e6f","pr_number":6998,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","signature_id":"2b92ea2834b8405b970ceee37d04d32b047f88649536fb5e87c255a7cf6b9715","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6886/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100566804978995200","occurred_at":"2026-09-17T12:45:27Z","signature_id":"4815c5ee092aaeb9f17a4f179de41c68c528994add0634df56951333903777fc","pr_number":6886,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","signature_id":"ece77cd8e3e47a8b500cc23dba72bce96033933da8cfc561a7bfe5d93a950f77","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"e8c855a1e7025053c74a73b5e48c4e6e96debc9ba1c7d6b424796483466cdf6f","pr_number":6933,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100560123062652928","occurred_at":"2026-09-17T12:18:54Z","signature_id":"777817b1e387b1f4492402d2f5429c721df7c24785215c5257845216b59cfccd","pr_number":6995,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6994/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100557383175507968","occurred_at":"2026-09-17T12:07:52Z","signature_id":"b66eb0c3d6894149e8e5bee7c3f68c812916f17da4df7663f06be67bb5aedc35","pr_number":6994,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","signature_id":"8d0248f7d9c1bf72afe316ba52f0dec7aa9b6c443c3506cf8e8d1a57be0d1c1b","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"f10ddf6a0df1f5dbb5292b3e49c11fae37f0c18b3ec87fcd7444bc0f39779449","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","signature_id":"e2639a0797c1c9a4a16eba734d19e610cbd443c3f33dd4bdd37f3d91b31ccf54","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"35bdd9c6de24d9640b64ae41e61a7affc59db651a49ce7a2fb22727cdcad6dc9","pr_number":6990,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6972/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100525488945500160","occurred_at":"2026-09-17T10:01:10Z","signature_id":"dff627a9d4028dda60dcb2cc608e3aeb26b1fb64501996026bbb8ff831d19570","pr_number":6972,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524896286150656","occurred_at":"2026-09-17T09:58:55Z","signature_id":"fd2143a50a59cd5afd961a3854a9a94e4c9bfcd849575bf08cd24c269273e2c6","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524822176993280","occurred_at":"2026-09-17T09:58:38Z","signature_id":"39a62cd962057ec142de0e390236317a312acabc0d41e0544a266cdbecf8c0b4","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523023395196928","occurred_at":"2026-09-17T09:51:23Z","signature_id":"360514ce5b258c6295b00ba9d2f1d4fd30911299b6a3d2dd1a60f05cf773940e","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100520057816748032","occurred_at":"2026-09-17T09:39:42Z","signature_id":"5472bf78238dce0fe2f94f0aff922f667d82ac0c02290e6e2cd57f8d35e3a169","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"a26f8c2c335980bba3f6e37f89e2b89d15ca7b79b6682f883c830ab2918e5b05","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100507818527297536","occurred_at":"2026-09-17T08:51:04Z","signature_id":"7e105769b349ece072f3c06670207d3d4017ec8870c2a43a5330028ceb5d52e3","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100506297609752576","occurred_at":"2026-09-17T08:45:01Z","signature_id":"962f9008502054736ab6ef9e552851e993e6894bbd6022a1a131a064d8689aee","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"d549df554d86a3fb7f1af2edc775ca43f25de1f5239534d866f56be388d7b3fb","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","signature_id":"371eb10e8c7a8df40477f826f6a0c01232597b190d53940fdbe6a8b4c627ede1","pr_number":6752,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","signature_id":"f82050e70a810fdc9a71c82fe836c8741b43edf42cceee0a76e6119d660eb530","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"826198aaa6fd1e4924bf9a5846057c446813d101971e8e7ab1f42a73a8fd3ad8","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100463746039156736","occurred_at":"2026-09-17T05:55:56Z","signature_id":"b8571782c9062ff08f362067402808930639b058a5f971bedc267ccc71e53617","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"082cd49a16ae701dbb38b1b79f0d1ac12e5a9df37271b1daf757d49fb6b996f0","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"3ddbbbbfcbe35abb46fc03a012581caf1092f9cbcef18f864c7ff0cb4097904d","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"e6916222b7c43fadfb77f5030602e037edec83d0ef773a05b13d08de9f0efc19","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":26,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":26,"after_last_push_count":26,"trend_counts":[0,0,0,0,0,7,26],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a440\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qr9h99g92762\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"name\": \"c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:28.114427313Z\",\n      \"endTime\": \"2026-09-17T02:31:33.96482384Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:108]: failed to create HCP cluster \"autoscaling-hcp-cluster\" with custom autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00110a440\u003e: \n    failed to create HCP cluster autoscaling-hcp-cluster: failed waiting for cluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-qr9h99g92762\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"name\": \"c2e04524-dc8f-4310-a549-fb6dc0199f5f\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:28.114427313Z\",\n      \"endTime\": \"2026-09-17T02:31:33.96482384Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0aba0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ms9kkl5qh4rm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"name\": \"7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:38.567378599Z\",\n      \"endTime\": \"2026-09-17T02:31:43.962353158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:88]: failed to create HCP cluster \"cilium-cl\" without CNI\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d0aba0\u003e: \n    failed to create HCP cluster cilium-cl: failed waiting for cluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-ms9kkl5qh4rm\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"name\": \"7a6f3465-1cd5-4662-8204-0e2dfd9ccda6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:38.567378599Z\",\n      \"endTime\": \"2026-09-17T02:31:43.962353158Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc740\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9lhfrbsqkhjk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"name\": \"39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:27.808727421Z\",\n      \"endTime\": \"2026-09-17T02:31:33.989605445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:92]: failed to create HCP cluster \"sre-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cdc740\u003e: \n    failed to create HCP cluster sre-hcp-cluster: failed waiting for cluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-9lhfrbsqkhjk\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"name\": \"39c45a7a-fb4f-41cf-82c7-e4d2fe160ef8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:02:27.808727421Z\",\n      \"endTime\": \"2026-09-17T02:31:33.989605445Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"02219053511d8a21950017349ee31494aa267bbe162808b309c406afe57bd88e","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"0ea6e92c609e741767496ed0bca205dd97df87334b05401151ca3d9e4297e4cc","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"139bcc210692fb4c4608f88c1e62ffbae5cc26a9ea38e8a550f86dc64be5bb37","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"2161ec92ec6ee1b1f4126d8043c75c164aa59ccae1d75ae8fdecc279478acefb","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"24e450fae1afaf3835ed114031825dba965e2638d9e2a4b78361d363bc7d139c","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"284fd36dc5ef5d74feffb6a93226bcfff835efdad8f1f85bdf032a8053cc1b6a","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"2e37c10dee44a98ad4c5d8e21f945e822bc6d1d157c0aec08182d653353bf301","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"2f9108fe18776541cbc85ef378b8f266052150ee76b5c630519c31e933ddb5b9","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"39e9738b3570d1ad49126bbe45de1c14c1222d36b148883b549374de80d01c71","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"417a674aaec0692852dbca6d977cc89fe23494c77a5a0606b6bafecdf7722e62","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"451e15ada92473b46c37ac7d85751c6c542d9156788736edb08510846c807aa5","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"593e3571cb87eb1c990363d1e184cfd091bfa920f1d8233ab9b8bffafc8753f4","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"626800ce28af5178db2305452aa40119998bcbdd63d53a7168853b1c7e730bd3","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"671e3e4773a45e7910b33f7d44f7a3f0d6be8188410cc6c10ced0d14da59716b","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"6d8549ce7ce3d738e38f7dd73b06fae91adfeb0e1534bba97833a20ae84d2f8a","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"7abcaf7cf828a86d797ada0bafb042a8494db58e91375d1805c53de72f0578cb","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"91e2917906e893c4adf51d8e3ad38930cff19ee5cb3009d04823d9b45fe41908","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9e699244c91afd46d8472b6ccd3aa4e2a0f99db87af440b007dcf8abcbb3ce6d","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9e87469877ef0917409c14d4d33b60f1b2ccdcdd81485077bc197c0d9d5a5e97","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"a8a2c2adbed4b3b26d51343db3e33c8589838c2cbac6661fcee12fbe4bf879fe","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"aa2abdeccf981853abdf8d84e7171612f5443e4af2a9fb66bb42c41ddd759ab7","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"c8695c56c9968b93a6495331c031bfc4f9defddf58bdeb3d72666b657cc7d8c5","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"d166da666940169c48a559831401ddc71f07bc674b8bac26199e6f32e66bd324","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"db18d58c7751ab0b22cdad498cec1c389a7f4bcbaed367974d93a33204cc2434","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"e19ccc38c497a63fa54dc54b1393fcc47c00e94ab08b2c73966853333434a251","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"f188a0c69f93a586d8064b19fecfad91c62cb72340482fa8197c45527b731e94","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":21,"runs_affected":21,"failed_runs":21,"run_impact_percent":10.714285714285714,"anchor_occurrences":21,"after_last_push_count":37,"trend_counts":[0,0,0,0,0,46,21],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":67,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":21}],"full_error_samples":["alert fired 7 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:00:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8t9zj\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-8t9zj (namespace arobit) on cluster ci01-j3118336-mgmt-1 has averaged 131.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:10:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-2\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-g6d4v\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-g6d4v (namespace arobit) on cluster ci01-j3118336-mgmt-2 has averaged 183.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:04:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b547f7d8b-bz92h\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-7b547f7d8b-bz92h (namespace mgmt-agent) on cluster ci01-j3118336-mgmt-1 has averaged 183.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:04:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c78bc956-rx5ct\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-86c78bc956-rx5ct (namespace clusters-service) on cluster ci01-j3118336-svc has averaged 829.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:08:29Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-vnkdq\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-vnkdq (namespace arobit) on cluster ci01-j3118336-mgmt-1 has averaged 166.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-17T08:54:27Z\n  Ended:       2026-09-17T09:09:27Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-mgmt-2\", component=\"service-memory\", container=\"mgmt-agent-controller\", namespace=\"mgmt-agent\", pod=\"mgmt-agent-7b547f7d8b-pnvkw\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container mgmt-agent-controller in pod mgmt-agent-7b547f7d8b-pnvkw (namespace mgmt-agent) on cluster ci01-j3118336-mgmt-2 has averaged 166.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-17T08:59:26Z\n  Ended:       2026-09-17T09:17:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3118336-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-86c78bc956-ljk5z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-86c78bc956-ljk5z (namespace clusters-service) on cluster ci01-j3118336-svc has averaged 226.4% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 3 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T08:17:39Z\n  Ended:       2026-09-17T08:27:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-49zq5\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-49zq5 (namespace arobit) on cluster ci01-j3085056-svc has averaged 168.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T08:17:39Z\n  Ended:       2026-09-17T08:27:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"aro-hcp-backend\", namespace=\"aro-hcp\", pod=\"aro-hcp-backend-d4b6fd9f9-p4w2z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container aro-hcp-backend in pod aro-hcp-backend-d4b6fd9f9-p4w2z (namespace aro-hcp) on cluster ci01-j3085056-svc has averaged 377.3% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T08:17:39Z\n  Ended:       2026-09-17T08:27:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j3085056-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-78587f7f-wtrr7\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-78587f7f-wtrr7 (namespace clusters-service) on cluster ci01-j3085056-svc has averaged 226.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T07:47:54Z\n  Ended:       2026-09-17T07:53:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-648575ffd8-pgfs2\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-648575ffd8-pgfs2 (namespace clusters-service) on cluster ci01-j6077952-svc has averaged 276.8% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T07:52:53Z\n  Ended:       2026-09-17T08:00:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-x8286\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-x8286 (namespace arobit) on cluster ci01-j6077952-mgmt-1 has averaged 128.1% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T07:54:53Z\n  Ended:       2026-09-17T08:01:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-mgmt-1\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-p74sm\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-p74sm (namespace arobit) on cluster ci01-j6077952-mgmt-1 has averaged 122.2% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.\n\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T08:17:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j6077952-svc\", component=\"service-memory\", container=\"clusters-service-server\", namespace=\"clusters-service\", pod=\"clusters-service-648575ffd8-526ht\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container clusters-service-server in pod clusters-service-648575ffd8-526ht (namespace clusters-service) on cluster ci01-j6077952-svc has averaged 185.9% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100485495183118336","occurred_at":"2026-09-17T07:22:21Z","signature_id":"f62c0a71de4ea91a4231cade3331db31d540a634ddef3ee99f2bb1c6a99266f1","pr_number":6989,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6791/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100479104343085056","occurred_at":"2026-09-17T06:56:58Z","signature_id":"aac5573ba90ecce2e869b424d24d0b2a557bdaf7fb68a59bdb1ca60dcfc91361","pr_number":6791,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100474592526077952","occurred_at":"2026-09-17T06:39:02Z","signature_id":"aa91dd60d940dac3239718e70007e00a82448bdb09061df222f973e65da4b8f6","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6880/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470565717413888","occurred_at":"2026-09-17T06:23:02Z","signature_id":"036570c8237e20bff362fa84f18b2757ecf6b7478b115ced0b1b6d96b52ae4ba","pr_number":6880,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100470315283910656","occurred_at":"2026-09-17T06:22:02Z","signature_id":"34004eeb935c0bb4ce4ba308740f33575b771c563e1c3c5cbd660527dc370ef4","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100469063225446400","occurred_at":"2026-09-17T06:17:04Z","signature_id":"07f59a2e15e5179fe3fd25749784f754d222157d8b27384b9e1cd64842ae8768","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100463746039156736","occurred_at":"2026-09-17T05:55:56Z","signature_id":"aaa081b0ec7f2d8d4a7e0e90f29ef2c663585ebb663aa2ffcb4b659e94e0c5f2","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100445405488615424","occurred_at":"2026-09-17T04:43:03Z","signature_id":"98874527ffbf8450aeb4dca36d3c24d835d1bfc8669e736b7a62e3aeb588719f","pr_number":6952,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100440653564481536","occurred_at":"2026-09-17T04:24:10Z","signature_id":"98473ea7cdc506ea6ed06c90a329992858dfe045473eb058bcb45937743f020e","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","signature_id":"d43422fbdada1a940ecd84da27859063da487a9ed357cd0dce3b03a5e1093c47","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100435900084260864","occurred_at":"2026-09-17T04:05:17Z","signature_id":"a15a05f0f8abcad7e64ec9115ec84f6ac3e540a459d0e285796dd856a7a6caa3","pr_number":6917,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100434265043898368","occurred_at":"2026-09-17T03:58:47Z","signature_id":"3e9ecaaa55541e660e55a54f116bbcb93077443ad73a39702bb7f87aa56a43f5","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","signature_id":"8469f16144ba929df5b9d7bb8ddc370454c226da3349d92d91c2c32c02a33ba7","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6439/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100415215534346240","occurred_at":"2026-09-17T02:43:05Z","signature_id":"05d61646d01952c54a7da4864113e0532f1dbe45108ad4402418ae2eaaf5a2b6","pr_number":6439,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100404999925272576","occurred_at":"2026-09-17T02:02:30Z","signature_id":"030419f084e0ac9f8645ed83bbb93e54a5eee29ea545c7c537285ba77f7a19f7","pr_number":6980,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"09d8e2ae3fbdc5cec5986413598d96675056845bffdde0b6a135d4a6c2fcafa1","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100399662610518016","occurred_at":"2026-09-17T01:41:17Z","signature_id":"5a274f8276581e7a1744e1e3283d6f882212c3ec0a22900f40bdbce973372f71","pr_number":6009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"5eff94552133a60d52e39851d75edd9d10579e18460fb3418389d4051065a47d","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388455866437632","occurred_at":"2026-09-17T00:56:45Z","signature_id":"f729f652c9b63e98893c72b91f3a1326eda28d14e829f566594109b7326b49cb","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"eeada1c413c3ca5fc2af1c6cbb216ce484bf15370729d3a7b144bcec22517b3c","pr_number":6894,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"a4314857c749da303216d314351149e0065b1d08354cb26b1fbedfc555fc6412","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"87cf86b25792a741c2861161a7d6340286a05f5515c53e2990a5c33b4b1e6faa","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":19,"runs_affected":4,"failed_runs":4,"run_impact_percent":2.0408163265306123,"anchor_occurrences":19,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,22,19],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":12,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001226768\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jwdh6h\" in resourcegroup=\"rg-candidate-4-22-9rtkhgwpwkdp\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\",\n      \"name\": \"497b563d-6a90-4107-8ea2-18e801b64cd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:09:44.365139487Z\",\n      \"endTime\": \"2026-09-17T13:33:50.717397424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7gelb6kbkk61h7dh9rg6bgds1mn2 still exists (deletion dispatched at 2026-09-17T13:09:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001226768\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-candidate-4-22-jwdh6h\" in resourcegroup=\"rg-candidate-4-22-9rtkhgwpwkdp\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/497b563d-6a90-4107-8ea2-18e801b64cd8\",\n      \"name\": \"497b563d-6a90-4107-8ea2-18e801b64cd8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:09:44.365139487Z\",\n      \"endTime\": \"2026-09-17T13:33:50.717397424Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp7gelb6kbkk61h7dh9rg6bgds1mn2 still exists (deletion dispatched at 2026-09-17T13:09:47Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001790120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8928t8c2w987\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n      \"name\": \"7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:04:21.545838494Z\",\n      \"endTime\": \"2026-09-17T13:28:30.638150402Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp95g2hh8pms8h7312ekq6dl4tr17v still exists (deletion dispatched at 2026-09-17T13:04:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001790120\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8928t8c2w987\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n      \"name\": \"7f319ad2-39f3-41e7-b4bb-2445dadbddeb\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:04:21.545838494Z\",\n      \"endTime\": \"2026-09-17T13:28:30.638150402Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp95g2hh8pms8h7312ekq6dl4tr17v still exists (deletion dispatched at 2026-09-17T13:04:23Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011b4600\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-8sx86dmkm86k\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13066ffb-8ec0-49b0-91b1-8f95cc778712\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13066ffb-8ec0-49b0-91b1-8f95cc778712\",\n      \"name\": \"13066ffb-8ec0-49b0-91b1-8f95cc778712\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:13:55.616199714Z\",\n      \"endTime\": \"2026-09-17T13:38:00.670848783Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp9797225e4pc4eg93kcbplgmr3noo still exists (deletion dispatched at 2026-09-17T13:14:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0011b4600\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"sre-hcp-cluster\" in resourcegroup=\"admin-api-breakglass-8sx86dmkm86k\" to finish deleting: GET https://rp.j4930944.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13066ffb-8ec0-49b0-91b1-8f95cc778712\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/13066ffb-8ec0-49b0-91b1-8f95cc778712\",\n      \"name\": \"13066ffb-8ec0-49b0-91b1-8f95cc778712\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:13:55.616199714Z\",\n      \"endTime\": \"2026-09-17T13:38:00.670848783Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2ssp9797225e4pc4eg93kcbplgmr3noo still exists (deletion dispatched at 2026-09-17T13:14:07Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"1200ec6f2e462f6a4057f0b6736f389e0eb4918e85be52611ba1988ed4a6b80f","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"26a26ee01c1fe63e9854b87ab08eead6a6fc7c9019b032530af1bdbbab01d85b","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"36c7d0ac2608fdd9b180eb76cfcef49dd063a5ddca7a5e1dd0eb82655e85e2bc","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"397c03aebb8f9974cabd4c075afe35800b5602c55edd622b828e4ee997bc99ca","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"51e487e0e9fc1b87185142de83749a342d6962b63a6106533902cf6918e54d5e","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"54d2d9e8120866d9194ddacd546d77becc75e8a251ac250bfb9fb8db3e040d85","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"83cf0cf00f201ed62094249bde1cad4326fd870d9fd95f573466f70f995ca9f3","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"a7c143432b9aa1db66f17c395896aa0a31866970f3d2152b89df2ba59a93d938","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"d5580f5abed7b32fb1241b3c5a25421aa30df617d15c7a847acda8ee9363e8f9","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"f1c4779490fb4bdafbfc35cb730136fb51e8d2baae0539cf8ed6ed1617f9f5b5","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"f1d1ec88637d584406805321e304585733a1d0d5f1dc372e90627e8f47025e62","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"1d5a773360bc54998f805181631ae95d9d4f7e700834cf8b2a7758ff8a01a8b3","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"5ffc110d49d5553b43d54e9686ac2a42d55e70924452ef37b6b6d552874166f9","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"8f0ccca9b1ab0f62cbe8877698c2eb88ab3fdc916c573bc80063829aa9d361db","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"d478c40d48df905f6ee8e97dc60c2a82d7080ab4ce2d7fdcba0c3903ce1b257b","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"f1fb92d8ffbf4155d66435595ff699141fa92d0cac0f75c60e648373d685e73b","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100553657568202752","occurred_at":"2026-09-17T11:53:12Z","signature_id":"e9fcf1ca8ec5a4e5b455439c42c7c5bdd9023eaf9099b980ad11a8e344c2334d","pr_number":6756,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"554758499da2dbd5ca2e6de15b8c5c7a3ee81c45e3edf3c5d71f98a2d63ec2c6","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"ff7aa9ff9517f007abe64da89b86cee0dfefa327cb83d202da998d1607130a76","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":7,"runs_affected":7,"failed_runs":7,"run_impact_percent":3.5714285714285716,"anchor_occurrences":7,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,8,7],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":7}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T21:49:48Z\n  Ended:       2026-09-17T22:07:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j1728768-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters has exceeded 250ms over the past 30 minutes (cluster ci01-j1728768-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T13:42:29Z\n  Ended:       2026-09-17T13:48:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j2532096-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j2532096-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T13:27:25Z\n  Ended:       2026-09-17T13:33:25Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9627648-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9627648-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","signature_id":"d9ede6c71c00bb7dbee4ddd3fb9aecba02c72aceca980c3acfa9826e45ff50ea","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6979/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100563400982532096","occurred_at":"2026-09-17T12:31:48Z","signature_id":"e9c0e585944f16bc753c7dace1676a24a15848f5a821759ef30ea4b61ef466bc","pr_number":6979,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554684329627648","occurred_at":"2026-09-17T11:56:52Z","signature_id":"8c32a70da3a734350e4252477cd5e4400a822058b8ff76cfef91cedb261d1712","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"a438005342e6e45a592ab0e52281d7a0356c34cf7a20d84dc4af128f220b1b66","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524826916556800","occurred_at":"2026-09-17T09:58:39Z","signature_id":"125c7452f650c84b80c4227b5aa637ed38ecce323a860ceae503938b79787746","pr_number":6986,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494087453937664","occurred_at":"2026-09-17T07:56:30Z","signature_id":"795144da2ef4d9ca390a9485406bde85898d43c117d008e0d213f31d6d5042d6","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6952/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488384702058496","occurred_at":"2026-09-17T07:33:50Z","signature_id":"77026658b7ddf2ead4aff5831cdfc9e1e01b461fa183028dccacbe6855fbe9a3","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":3.061224489795918,"anchor_occurrences":6,"after_last_push_count":7,"also_in":["PROD","STG"],"trend_counts":[0,0,0,0,0,6,6],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":12,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:124]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c52c40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-s2zdnnlcz4wt\" to finish creating: GET https://rp.j5719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80008e-4b66-48be-b60f-ca94f871de32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80008e-4b66-48be-b60f-ca94f871de32\",\n      \"name\": \"1a80008e-4b66-48be-b60f-ca94f871de32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:56:32.579892703Z\",\n      \"endTime\": \"2026-09-17T22:15:39.575240678Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:124]: failed to create node pool \"np-1\" for private KAS cluster \"private-kas\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c52c40\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kas\" in resourcegroup=\"private-kas-s2zdnnlcz4wt\" to finish creating: GET https://rp.j5719424.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80008e-4b66-48be-b60f-ca94f871de32\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1a80008e-4b66-48be-b60f-ca94f871de32\",\n      \"name\": \"1a80008e-4b66-48be-b60f-ca94f871de32\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:56:32.579892703Z\",\n      \"endTime\": \"2026-09-17T22:15:39.575240678Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8c920\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-z9b9d4ng6qd2\" to finish creating: GET https://rp.j2375680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\",\n      \"name\": \"ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:14:44.291403118Z\",\n      \"endTime\": \"2026-09-17T21:33:45.719372752Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:123]: failed to create node pool ea-np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000c8c920\u003e: \n    failed to create NodePool ea-np-1: failed waiting for nodepool=\"ea-np-1\" for cluster \"ea-cluster\" in resourcegroup=\"external-auth-cluster-z9b9d4ng6qd2\" to finish creating: GET https://rp.j2375680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\",\n      \"name\": \"ad30d90d-8aef-4edc-88c0-22bb42dbb8f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:14:44.291403118Z\",\n      \"endTime\": \"2026-09-17T21:33:45.719372752Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8b160\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-zj6grj\" in resourcegroup=\"rg-np-version-upgrade-zj6grj-pp9qwfhhwgc8\" to finish creating: GET https://rp.j3847296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85af6203-800d-4bc5-a875-4385db2526c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85af6203-800d-4bc5-a875-4385db2526c1\",\n      \"name\": \"85af6203-800d-4bc5-a875-4385db2526c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T11:29:25.794778495Z\",\n      \"endTime\": \"2026-09-17T11:48:27.334521515Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d8b160\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-zj6grj\" in resourcegroup=\"rg-np-version-upgrade-zj6grj-pp9qwfhhwgc8\" to finish creating: GET https://rp.j3847296.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85af6203-800d-4bc5-a875-4385db2526c1\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/85af6203-800d-4bc5-a875-4385db2526c1\",\n      \"name\": \"85af6203-800d-4bc5-a875-4385db2526c1\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T11:29:25.794778495Z\",\n      \"endTime\": \"2026-09-17T11:48:27.334521515Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691156575719424","occurred_at":"2026-09-17T20:59:35Z","signature_id":"13adef91a90ab725dab440fb0cd3d153b3bf6f23ccf1898e5f571b9c23787d80","pr_number":6939,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100678231362375680","occurred_at":"2026-09-17T20:03:18Z","signature_id":"14578415f39945108c1077bc9568d80eb9c57e111ae4ccc29677f2723df82376","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6985/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100532310913847296","occurred_at":"2026-09-17T10:28:23Z","signature_id":"11862ebb0b64914c35ffbbc35bccd2d2fc728333e2922d73613bd21bacfcabf1","pr_number":6985,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6879/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523779246526464","occurred_at":"2026-09-17T09:54:29Z","signature_id":"7fc0697aa9693154afc06e12851d0da5c2c6857a0d678a54183094477e417af7","pr_number":6879,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494069930135552","occurred_at":"2026-09-17T07:56:26Z","signature_id":"41623d8231704f6b3918018ef069742c2b6b4139b736f8af0a971c826a53f099","pr_number":6940,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100469063225446400","occurred_at":"2026-09-17T06:17:04Z","signature_id":"41037677a8ddb37389a2d414387cf88a833a474c6bb9aecf6b5fe7288cd9a6b8","pr_number":6986,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":6,"runs_affected":6,"failed_runs":6,"run_impact_percent":3.061224489795918,"anchor_occurrences":6,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,6,6],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":12,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","occurrences":6}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T22:35:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6073984-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.6:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6073984/providers/microsoft.monitor/accounts/hcps-j6073984\", namespace=\"ocm-arohcpci01-2st0r3e7gsp00p1v6je2veqpisj6tv8b-a8q3g3x9d3e6x7g\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-7mb8q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2st0r3e7gsp00p1v6je2veqpisj6tv8b-a8q3g3x9d3e6x7g/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T22:35:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j6073984-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.6:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6073984/providers/microsoft.monitor/accounts/hcps-j6073984\", namespace=\"ocm-arohcpci01-2st0r3e7gsp00p1v6je2veqpisj6tv8b-a8q3g3x9d3e6x7g\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-7mb8q\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2st0r3e7gsp00p1v6je2veqpisj6tv8b-a8q3g3x9d3e6x7g/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T11:12:18Z\n  Ended:       2026-09-17T11:18:19Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j5196928-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"router\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.120:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5196928/providers/microsoft.monitor/accounts/hcps-j5196928\", namespace=\"ocm-arohcpci01-2ssn9fqkkgqtkcb1te9l2nsihdvccpop-x8g8g9w0z4n5j0k\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-hxhtp\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2ssn9fqkkgqtkcb1te9l2nsihdvccpop-x8g8g9w0z4n5j0k/router has not matched the expected number of replicas for longer than 30 minutes.","alert fired 10 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T09:55:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T10:06:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T10:16:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T10:16:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T10:19:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.37:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv6ma6mn5k5mt36luuom09289pq58-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-v4f24\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv6ma6mn5k5mt36luuom09289pq58-cilium-cl/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T10:19:32Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.37:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv6ma6mn5k5mt36luuom09289pq58-cilium-cl\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-v4f24\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv6ma6mn5k5mt36luuom09289pq58-cilium-cl/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T10:24:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T10:24:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"cluster-api\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r/cluster-api has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T10:24:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv0qgf0brht8l82lblg9u4aedu59m-l0t6g2b3x9j1i5r/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T10:24:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j0442496-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.4:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j0442496/providers/microsoft.monitor/accounts/hcps-j0442496\", namespace=\"ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-fstb6\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2sslv1q0m5l3l4h7q2qhv5phss5tml71-w3t5e3f8l5j8d5r/capi-provider has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","signature_id":"a25eab3d2aea0534a58060ed60f2bfb99bd3c649f3ff0b27d214e0eebb1f8f68","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100523023395196928","occurred_at":"2026-09-17T09:51:23Z","signature_id":"98cf6a34d877442691cd61bb1e77f0e720f13f9f1c1c8cdddc6ad3341afc78ff","pr_number":5724,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"a85ef3a241aa9fee6ef7ff6043d1020492ba3c7fed3de7483ade938f342c7c3e","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"a995d8a36ccb5cf52cd9d35f8ae8324aa7b3c5eb47ebb28e229463bfb8fb04c6","pr_number":6988,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"6133b883dec1a7ae7199f6897e93bfe8938d4615907efd1f6e96f8f531de45b7","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100385816462233600","occurred_at":"2026-09-17T00:46:16Z","signature_id":"54b42f0c1ae5cc077af2dfa797005520003ed2bd8ca931aabcdf79c00d97c263","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"343646bfd94354940e62f77c67e650c2e3e31608f42bb5379cf21e51febcdd6a","failure_pattern":"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","search_query":"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":2.5510204081632653,"anchor_occurrences":5,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,10,5],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":15,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":2},{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=2)","occurrences":3}],"full_error_samples":["time=2026-09-17T22:08:23.042Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T22:08:24.010Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-17T22:08:24.819Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=949ef12d6ab429b901225672cf02dc8c2e8f5344e8e4f364f694e8721bc1ed4e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0316160-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F949ef12d6ab429b901225672cf02dc8c2e8f5344e8e4f364f694e8721bc1ed4e\ntime=2026-09-17T22:38:22.805Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-17T21:21:29.249Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T21:21:29.969Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2\ntime=2026-09-17T21:21:30.723Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 deployment=63b12b492d5d28b0b0b18408fa469c00c31993867db99322c5926f55f6db447a portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6654848-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F63b12b492d5d28b0b0b18408fa469c00c31993867db99322c5926f55f6db447a\ntime=2026-09-17T21:51:28.943Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-17T21:19:38.472Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T21:19:39.145Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-17T21:19:39.871Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=f518dca3cc0829098231fe8a127f84b76b966486264714a9e41a0d1fd09997db portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j0025600-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ff518dca3cc0829098231fe8a127f84b76b966486264714a9e41a0d1fd09997db\ntime=2026-09-17T21:49:38.148Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6805/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100701574740316160","occurred_at":"2026-09-17T21:40:59Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6805,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100691038216654848","occurred_at":"2026-09-17T20:59:07Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6964,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100689397040025600","occurred_at":"2026-09-17T20:52:35Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6945,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495912190087168","occurred_at":"2026-09-17T08:03:45Z","signature_id":"02196de5a476d3b1ee4c19a8bc0f226a6c225ebce4617f1571165cf2a88ff621","pr_number":6688,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6980/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488996214804480","occurred_at":"2026-09-17T07:36:16Z","signature_id":"4ff50d3e282718dc8a8b59118c6ea7b7ffbd56f6406c52bebb57aea88b24fbf4","pr_number":6980,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.530612244897959,"anchor_occurrences":3,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,5,3],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001d94018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-g7cwbl5k67j4\" to finish creating: GET https://rp.j8532992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"name\": \"fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T00:05:39.3471997Z\",\n      \"endTime\": \"2026-09-18T00:06:19.6272248Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:403]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc001d94018\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: no cilium pods found in kube-system namespace\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-g7cwbl5k67j4\" to finish creating: GET https://rp.j8532992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"name\": \"fbeee8b5-27cd-473e-a9c2-df2114c56f2b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-18T00:05:39.3471997Z\",\n      \"endTime\": \"2026-09-18T00:06:19.6272248Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00105a180\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-l59lxtj9lwhd\" to finish creating: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7095545b-bbd0-41f6-961e-6bb475b19413\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7095545b-bbd0-41f6-961e-6bb475b19413\",\n      \"name\": \"7095545b-bbd0-41f6-961e-6bb475b19413\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:01:16.192258014Z\",\n      \"endTime\": \"2026-09-17T13:01:58.244062799Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:198]: failed to create node pool version 4.20.32 for back-level cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00105a180\u003e: \n    failed waiting for nodepool=\"np-ver-4-20-32\" for cluster \"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-l59lxtj9lwhd\" to finish creating: GET https://rp.j7653760.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7095545b-bbd0-41f6-961e-6bb475b19413\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7095545b-bbd0-41f6-961e-6bb475b19413\",\n      \"name\": \"7095545b-bbd0-41f6-961e-6bb475b19413\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:01:16.192258014Z\",\n      \"endTime\": \"2026-09-17T13:01:58.244062799Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8f5e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-n2zmdkblldlw\" to finish creating: GET https://rp.j0741376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cfa413af-9795-4d14-a521-6638502432ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cfa413af-9795-4d14-a521-6638502432ab\",\n      \"name\": \"cfa413af-9795-4d14-a521-6638502432ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T04:05:39.31416638Z\",\n      \"endTime\": \"2026-09-17T04:06:12.119697969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/simple_negative_cases.go:159]: failed to create node pool np-1\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b8f5e0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"negative-tests-cluster\" in resourcegroup=\"rg-negative-tests-n2zmdkblldlw\" to finish creating: GET https://rp.j0741376.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cfa413af-9795-4d14-a521-6638502432ab\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/cfa413af-9795-4d14-a521-6638502432ab\",\n      \"name\": \"cfa413af-9795-4d14-a521-6638502432ab\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T04:05:39.31416638Z\",\n      \"endTime\": \"2026-09-17T04:06:12.119697969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721125368532992","occurred_at":"2026-09-17T22:58:38Z","signature_id":"3ad0c7f8513c2d305c0610677386a9460eaa446d3f7dfb59c408d79cec65a7ec","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6974/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554687697653760","occurred_at":"2026-09-17T11:56:54Z","signature_id":"0e7b7131df9efbbc641dfa4dfd266b388acf38942df6680d264c1a305036569d","pr_number":6974,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100420606720741376","occurred_at":"2026-09-17T03:04:31Z","signature_id":"4b99cbe35f0bd3c6197a97e4f313692f0f80c116d5fba8a8d582ebf622976c32","pr_number":6948,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":1.530612244897959,"anchor_occurrences":3,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,4,3],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":7,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.001s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6856,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"5ce02ffd73e5645f7d2b83086ab3bc44a214a4293b62aab409040cc411cfb879","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b414f524aec7c4ea3c6d464450b9af6f734740715a61aff1c3a8f53547f62641","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.0204081632653061,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,3,2],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000648280\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qghsblcj4pgk\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"name\": \"f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:29:11.075687263Z\",\n      \"endTime\": \"2026-09-17T23:58:14.591633856Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:96]: failed to create HCP cluster \"v20260901\" via v20260901preview\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000648280\u003e: \n    failed to create HCP cluster v20260901: failed waiting for cluster=\"v20260901\" in resourcegroup=\"v20260901preview-qghsblcj4pgk\" to finish creating: GET https://rp.j4009472.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"name\": \"f0b8c439-b7ef-4e73-b372-5a8c55bbf10e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:29:11.075687263Z\",\n      \"endTime\": \"2026-09-17T23:58:14.591633856Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ec480\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4llzpkxqdt5h\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"name\": \"98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:29.574806708Z\",\n      \"endTime\": \"2026-09-17T13:41:32.177488403Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:98]: failed to create HCP cluster \"private-kv-cluster\" with private keyvault\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015ec480\u003e: \n    failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-4llzpkxqdt5h\" to finish creating: GET https://rp.j2573056.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"name\": \"98da8819-6ed6-431a-b25f-78eab8ef8dd9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:29.574806708Z\",\n      \"endTime\": \"2026-09-17T13:41:32.177488403Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: router; hosted cluster degraded: UnavailableReplicas: router deployment has 2 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712773964009472","occurred_at":"2026-09-17T22:25:29Z","signature_id":"3952b4207ca326cafd047aae37e1541b9c1921b3b3a1de670914b6430814079b","pr_number":6999,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554680982573056","occurred_at":"2026-09-17T11:56:43Z","signature_id":"d7b3a6738a3142cf28e35b425618e47c2d78a5ec741df6019ce6ae2b560d9004","pr_number":6990,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.0204081632653061,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,2,2],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":2}],"full_error_samples":["alert fired 5 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T03:10:30Z\n  Ended:       2026-09-17T03:22:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"config-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/config-policy-controller has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T03:10:30Z\n  Ended:       2026-09-17T03:22:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T03:10:30Z\n  Ended:       2026-09-17T03:22:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T03:11:28Z\n  Ended:       2026-09-17T03:21:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"governance-policy-framework\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/governance-policy-framework has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-17T03:11:28Z\n  Ended:       2026-09-17T03:21:28Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j9076992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9076992/providers/microsoft.monitor/accounts/services-j9076992\", namespace=\"klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-6q7r5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssg777qqnq4cjon6gmqfatr43murnr3/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.","alert fired 16 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:29:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:30:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T02:35:35Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfodlh52rjsn8v0i6tqm6t35c2l0p0\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfodlh52rjsn8v0i6tqm6t35c2l0p0/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T02:35:35Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfoc87t8ajfhu9n4pqjtbphn862gj1\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfoc87t8ajfhu9n4pqjtbphn862gj1/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 5:\n  State:       Resolved\n  Started:     2026-09-17T02:35:35Z\n  Ended:       2026-09-17T02:42:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfoc8cioja4t99v8qf8fgteg8hp8cq\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfoc8cioja4t99v8qf8fgteg8hp8cq/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 6:\n  State:       Resolved\n  Started:     2026-09-17T02:35:35Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfofif68524qhonc25k6st2irl6nvr\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfofif68524qhonc25k6st2irl6nvr/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 7:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfodboatgvtrijvb9itq8g6nv3a0lj\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfodboatgvtrijvb9itq8g6nv3a0lj/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 8:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfod4unnpb3fidghon0kvgohtrm0gi\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfod4unnpb3fidghon0kvgohtrm0gi/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 9:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfod7a9s6uqchgri0b4afl9dc7n19f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfod7a9s6uqchgri0b4afl9dc7n19f/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 10:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfoda33s4gbilpj50f3r2392mijctf\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfoda33s4gbilpj50f3r2392mijctf/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 11:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:43:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfodmtcgprbt25mkk7n1f31gp57v5p\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfodmtcgprbt25mkk7n1f31gp57v5p/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 12:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:59:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 13:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfofsi9cr1hrnf33j5m2j2v0anob00\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfofsi9cr1hrnf33j5m2j2v0anob00/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 14:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:54:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 15:\n  State:       Resolved\n  Started:     2026-09-17T02:36:36Z\n  Ended:       2026-09-17T02:42:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfof0kh0mr490ilc2tusdfihbkudsb\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfof0kh0mr490ilc2tusdfihbkudsb/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 16:\n  State:       Resolved\n  Started:     2026-09-17T02:37:34Z\n  Ended:       2026-09-17T02:43:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfohaljjdbuho1cfahj1o9d3dj04u4\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2ssfohaljjdbuho1cfahj1o9d3dj04u4/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"69b255fd750c54b7a1baf2eca24627dd8e61f4fdef893acd92470a12832db0f6","pr_number":6976,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"48e28661b4bfc093fce3e4eccd4ac9c8f3996f35cf8b87e4d8ccb1de279e00a8","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.0204081632653061,"anchor_occurrences":2,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,4,2],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-18T01:12:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j6258560-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-m8n67\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-m8n67 (namespace arobit) on cluster ci01-j6258560-svc has averaged 125.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T14:18:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j7954560-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-th462\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-th462 (namespace arobit) on cluster ci01-j7954560-svc has averaged 125.4% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100721091256258560","occurred_at":"2026-09-17T22:58:32Z","signature_id":"fffbb7c99a9e3fbfbbfb27296f90fe1eaf381e133c47c2097dc7c5b3440df9c4","pr_number":7000,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554691157954560","occurred_at":"2026-09-17T11:57:00Z","signature_id":"f1e05c2e51f466f034476da1ae61bfe77799ca5b43dd98ad6f2836fda4da614d","pr_number":6752,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5635348e73941180a257b29f206dcd0308f769e5208af4b6eec6ee1b4f601a83","failure_pattern":"client rate limiter Wait returned an error: context deadline exceeded","search_query":"client rate limiter Wait returned an error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.0204081632653061,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014180c0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-phbtk2 namespace are ready: failed to list pods in cilium-connectivity-check-phbtk2 namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014180c0\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-phbtk2 namespace are ready: failed to list pods in cilium-connectivity-check-phbtk2 namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168f0c8\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-txdwxr namespace are ready: failed to list pods in cilium-connectivity-check-txdwxr namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:407]: failed to run simple web app and connectivity check app with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00168f0c8\u003e: \n    VerifyCiliumConnectivityChecks failed: connectivity check failed, not all pods in cilium-connectivity-check-txdwxr namespace are ready: failed to list pods in cilium-connectivity-check-txdwxr namespace: client rate limiter Wait returned an error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6948/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100434265043898368","occurred_at":"2026-09-17T03:58:47Z","signature_id":"297e17fd0f8b231f49a6a64a53967d8a3faf0348a5fdc0a0860997681f536656","pr_number":6948,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6976/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100402745189076992","occurred_at":"2026-09-17T01:53:32Z","signature_id":"2bd939aaad187ab78804671b6970c3fb300c0b1bc1be22955c06bc837de68bb9","pr_number":6976,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c71ebc2a93d9fc10a75dcaf39a2b1325de6379c3891238b64d1e4e8f4281df9a","failure_pattern":"failed to get kusto cluster \u003ckusto-cluster\u003e from config: configuration: key kusto not found","search_query":"failed to get kusto cluster name from config: configuration: key kusto not found","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.0204081632653061,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,2],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000aac060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc000aac060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0018b8060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:101]: Timed out after 600.007s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0018b8060\u003e: \n    failed to get kusto cluster name from config: configuration: key kusto not found\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","signature_id":"99c5a5b19f403073edd6e8406f2272a3fda7ff44e9a289f5c999503a39917368","pr_number":6672,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"daa3fa7c154560025dfba9dba52f2111304f2d047aafee6e951b7d85e52a0fdc","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.0204081632653061,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f1dd40\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f1dd40\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-xrnlcrchtggh\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPCluster20260901AndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-xrnlcrchtggh, error: context deadline exceeded\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014fc8a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zvg8dvqqf6zz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zvg8dvqqf6zz, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0014fc8a0\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-zvg8dvqqf6zz\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-zvg8dvqqf6zz, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"d7f2fd6c375699dace4737f150cc1d0c82379eed84d0d265ff57b17f49ba73d8","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"a4ab9812dd60d9325a7127735e40ad640073cecf361f77bd3c75a60c7b17bd98","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"68912772397b319aad954546c15ea4edddfc908ce0b698d54b06e7d4df6b6ccf","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: serviceProviderClusters; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,3,2],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00112c090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-8bctrm\" in resourcegroup=\"rg-zstream-upgrade-4-20-2lc8kzcjbshc\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83ec0b10-2ee0-45fb-aaa3-b482a6f739da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83ec0b10-2ee0-45fb-aaa3-b482a6f739da\",\n      \"name\": \"83ec0b10-2ee0-45fb-aaa3-b482a6f739da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:36:14.84381025Z\",\n      \"endTime\": \"2026-09-17T10:00:14.610296792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv1q0m5l3l4h7q2qhv5phss5tml71 still exists (deletion dispatched at 2026-09-17T09:36:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00112c090\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cluster-zstream-4-20-8bctrm\" in resourcegroup=\"rg-zstream-upgrade-4-20-2lc8kzcjbshc\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83ec0b10-2ee0-45fb-aaa3-b482a6f739da\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/83ec0b10-2ee0-45fb-aaa3-b482a6f739da\",\n      \"name\": \"83ec0b10-2ee0-45fb-aaa3-b482a6f739da\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:36:14.84381025Z\",\n      \"endTime\": \"2026-09-17T10:00:14.610296792Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv1q0m5l3l4h7q2qhv5phss5tml71 still exists (deletion dispatched at 2026-09-17T09:36:15Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0025dc138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2wlqnnchcnfz\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06cacc7f-3109-44fc-aa1f-226b6d9287df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06cacc7f-3109-44fc-aa1f-226b6d9287df\",\n      \"name\": \"06cacc7f-3109-44fc-aa1f-226b6d9287df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:13:15.932719086Z\",\n      \"endTime\": \"2026-09-17T09:37:24.699046818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv0qgf0brht8l82lblg9u4aedu59m still exists (deletion dispatched at 2026-09-17T09:13:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0025dc138\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-2wlqnnchcnfz\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06cacc7f-3109-44fc-aa1f-226b6d9287df\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/06cacc7f-3109-44fc-aa1f-226b6d9287df\",\n      \"name\": \"06cacc7f-3109-44fc-aa1f-226b6d9287df\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:13:15.932719086Z\",\n      \"endTime\": \"2026-09-17T09:37:24.699046818Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv0qgf0brht8l82lblg9u4aedu59m still exists (deletion dispatched at 2026-09-17T09:13:16Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"56f9b496d774ec73f62dc8457705f5fe010fad5d913e37425e1aeca77033669a","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"e88abeefa8796da83e3bcaacbe5d5b7d5cc078408bb87028a285969eda618a61","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4f70c086dc5495ea728fc39aab23a54b6cce4072ba8ccfc36ee6980300ba5299","failure_pattern":"CreateOrUpdateExternalAuthAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002990240\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:276]: failed to create external auth config \"ext-auth-cidr\"\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002990240\u003e: \n    failed waiting for external auth \"ext-auth-cidr\" in resourcegroup=\"e2e-cidr-connectivity-j7tx89s2rwk5\" for cluster=\"cidr-connectivity-test\" to finish creating or updating, caused by: timeout '15.000000' minutes exceeded during CreateOrUpdateExternalAuthAndWait for external auth ext-auth-cidr in cluster cidr-connectivity-test in resource group e2e-cidr-connectivity-j7tx89s2rwk5, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","signature_id":"bf84ee4449e687d0ad4cd4859ef7515e9901f5ded80364d34ebb748aaa5b41df","pr_number":6999,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"87788fb66fa66f5634c4510270d5e7da1c08535316668adf092302880431221e","failure_pattern":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle: Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,2,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:158]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-prgsd6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106cb20\u003e: \n    DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:188]: failed to verify simple web app runs on cluster \"cluster-candidate-4-20-prgsd6\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106cb20\u003e: \n    DNS for route host agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-jbbn8.apps.aro.b9d8s7t1n3o1u4s.irz8.j8665344.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5724/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100524951428665344","occurred_at":"2026-09-17T09:59:04Z","signature_id":"b9c07fb6e82776d012cae86cfc979add2b4b19f7b344021874689105ee18c987","pr_number":5724,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"293006e7a1ad208844fa10f4e2b3c0c8959662e799277a9c1d23e6534078c9e0","failure_pattern":"DeleteHCPCluster timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a16108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc001a16108\u003e: \n    failed to cleanup resource group: failed deleting hcp clusters in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\", caused by: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded, error: failed waiting for hcpCluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-q8l56f6rlm84\" to finish deleting, caused by: timeout '25.000000' minutes exceeded during DeleteHCPCluster for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-q8l56f6rlm84, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554765694930944","occurred_at":"2026-09-17T11:57:31Z","signature_id":"e37087f31e7094fe45c7ae2868faf05fde32666577b47902fe94fbe270287c3b","pr_number":6993,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"54082ad8881c5131bff996f7153979c81faa7fa74e3b195d969ebc6eb63a2c19","failure_pattern":"ERROR CODE: DeploymentFailed; detail code DeploymentActive; detail message The deployment with resource id '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Resources/deployments/cluster-\u003cid\u003e' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '\u003ctimestamp\u003e' with correlationId '\u003cuuid\u003e', and will expire at '\u003ctimestamp\u003e' if it does not complete before then.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster (stamp=1)","occurrences":1}],"full_error_samples":["time=2026-09-17T08:52:09.303Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/mgmt-cluster.bicep\\n  Parameters: configurations/mgmt-cluster.tmpl.bicepparam\"\ntime=2026-09-17T08:52:10.143Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1\ntime=2026-09-17T08:52:11.020Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 deployment=e5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j2728320-mgmt-1%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8\ntime=2026-09-17T08:54:11.408Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster stamp=1 err=\"stamp 1: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2728320-mgmt-1/providers/Microsoft.Resources/deployments/e5c452463d5a2ca118d246d0e10c4f76fe32f1478279da2b4450028ca1d7e3f8/operationStatuses/08584119717548817375\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentActive\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The deployment with resource id '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j2728320-mgmt-1/providers/Microsoft.Resources/deployments/cluster-34wn2l5qqmupi' cannot be saved, because this would overwrite an existing deployment which is still active. If this is an intentional re-deployment, please wait for the original deployment to complete, or cancel it before re-attempting the operation. The previous deployment was started at '9/17/2026 8:52:28 AM' with correlationId 'd7f46ae1-ae89-4906-81b7-ab9a15265766', and will expire at '9/24/2026 8:52:28 AM' if it does not complete before then. Please see https://aka.ms/arm-deploy-resources for usage details.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100500788332728320","occurred_at":"2026-09-17T08:23:07Z","signature_id":"9fef102a68f061bbb39fd685e7e0d217cf4fce57b870517975d3d5e21e3e90e2","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"46425ffc9312d96c5106d1b27946c7775e4ddf6d0471e792a3aaf8c2bc4d3a86","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc120\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w5shsm9ng8kc\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"name\": \"eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T04:52:38.625826904Z\",\n      \"endTime\": \"2026-09-17T05:21:45.494759028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 2m17s ago; version 4.22.14 is Partial (want Completed), started 5m22s ago; version 4.22.14 is Partial (want Completed), started 12m16s ago; version 4.22.14 is Partial (want Completed), started 15m20s ago; version 4.22.14 is Partial (want Completed), started 17m23s ago; version 4.22.14 is Partial (want Completed), started 17m58s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fbc120\u003e: \n    failed to create HCP cluster private-kas: failed waiting for cluster=\"private-kas\" in resourcegroup=\"private-kas-w5shsm9ng8kc\" to finish creating: GET https://rp.j5021696.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"name\": \"eb7aec2b-15f3-48ae-bf39-e0b303d7f6f8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T04:52:38.625826904Z\",\n      \"endTime\": \"2026-09-17T05:21:45.494759028Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.22.14 is Partial (want Completed), started 2m17s ago; version 4.22.14 is Partial (want Completed), started 5m22s ago; version 4.22.14 is Partial (want Completed), started 12m16s ago; version 4.22.14 is Partial (want Completed), started 15m20s ago; version 4.22.14 is Partial (want Completed), started 17m23s ago; version 4.22.14 is Partial (want Completed), started 17m58s ago; hosted cluster degraded: UnavailableReplicas: openshift-apiserver deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6986/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100438339525021696","occurred_at":"2026-09-17T04:14:58Z","signature_id":"421f5b6d838c121843fe235c14d4d70af34427ca212e0849c68ed8aa29654d6b","pr_number":6986,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"9cb32ab26385522cb2e4937166fa88ab05c33466c0371048c193b661db0d935d","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,6,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a10e80\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8nfj4dg6xpxk\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"name\": \"063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:48:07.392021884Z\",\n      \"endTime\": \"2026-09-17T03:17:09.326897061Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a10e80\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8nfj4dg6xpxk\" to finish creating: GET https://rp.j6537856.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"name\": \"063e2ac2-9588-4f9c-82b9-899a69317856\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:48:07.392021884Z\",\n      \"endTime\": \"2026-09-17T03:17:09.326897061Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6894/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100386114316537856","occurred_at":"2026-09-17T00:47:27Z","signature_id":"d7f93e547e4be2029c411ab0ce7110ad6d4b2c44c4f85c58ccad50e27c80ca41","pr_number":6894,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"fe6b01560ac49646fd0a46c48f672cf2fc0ebbf844ea4576bb0cab7a46df7ce8","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,4,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90a40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8x87dzwxvb96\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"name\": \"3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:04:32.691734276Z\",\n      \"endTime\": \"2026-09-17T02:33:33.968010236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:207]: failed to create HCP cluster \"agg-cluster\" with aggregated settings\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f90a40\u003e: \n    failed waiting for cluster=\"agg-cluster\" in resourcegroup=\"feature-aggregation-8x87dzwxvb96\" to finish creating: GET https://rp.j7881088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"name\": \"3edfae6e-fb2d-4e52-a859-1500f9def9f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T02:04:32.691734276Z\",\n      \"endTime\": \"2026-09-17T02:33:33.968010236Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KASLoadBalancerNotReachable: APIServer external route not admitted\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9020e6f833dc38c47ccef3d4182afa72cebc722f85d46ac12ebd56299acce896","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f2cdc20003f8bc1785ec3c6201141b4cb36db8aaf93567e547373b8f3bb93ad9","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,7,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":8,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4c440\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xfhpjp2z56q6\" to finish creating: GET https://rp.j9085312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\",\n      \"name\": \"0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T07:37:20.85368782Z\",\n      \"endTime\": \"2026-09-17T08:06:23.778430257Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:161]: failed to create HCP cluster for patch-tags test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f4c440\u003e: \n    failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-xfhpjp2z56q6\" to finish creating: GET https://rp.j9085312.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\",\n      \"name\": \"0a0c90ee-9e20-4f29-aca3-8f6c86d2ab47\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T07:37:20.85368782Z\",\n      \"endTime\": \"2026-09-17T08:06:23.778430257Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"ebdc66289f4002396961893fc85f6cbf2d72c402de90efcaab065108518df75a","pr_number":6988,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"433be7606ae055f59c8cb57591e36f85c0ec2b280bd34579035b9bea62b92600","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0004ec1e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hdsrgp87vhp6\" to finish deleting: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"name\": \"140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:51:08.271574522Z\",\n      \"endTime\": \"2026-09-17T22:15:12.38485138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st0r3e7gsp00p1v6je2veqpisj6tv8b still exists (deletion dispatched at 2026-09-17T21:51:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0004ec1e0\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"autoscaling-hcp-cluster\" in resourcegroup=\"autoscaling-cluster-hdsrgp87vhp6\" to finish deleting: GET https://rp.j6073984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"name\": \"140fc1a6-e41e-4f25-9908-25d174a04842\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T21:51:08.271574522Z\",\n      \"endTime\": \"2026-09-17T22:15:12.38485138Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2st0r3e7gsp00p1v6je2veqpisj6tv8b still exists (deletion dispatched at 2026-09-17T21:51:09Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7000/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100685654236073984","occurred_at":"2026-09-17T20:37:43Z","signature_id":"fa245128857c23599fcf8f3484b06ab6ca5868cca2b19c36072deb31d3760300","pr_number":7000,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"82fcc2cca51beebf00c7302e8f8dc3563b7995a4550dbcf53d6d84d1820ed342","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster still exists; [clusterServiceStatus] state is \"uninstalling\"; [hostedCluster] HostedCluster still exists; [resourceTeardown] tearing down namespaces; [descendantResources] remaining resources: nodePools, serviceProviderClusters, serviceProviderNodePools; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a2b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fth2ns8rql9z\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"name\": \"e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:27:47.803109699Z\",\n      \"endTime\": \"2026-09-17T09:51:54.618583456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv6ma6mn5k5mt36luuom09289pq58 still exists (deletion dispatched at 2026-09-17T09:27:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc00101a2b8\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"cilium-cl\" in resourcegroup=\"cni-cilium-fth2ns8rql9z\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"name\": \"e8997c6b-6d2a-4f02-afe2-3927067cda87\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:27:47.803109699Z\",\n      \"endTime\": \"2026-09-17T09:51:54.618583456Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline; [clusterServiceDeletion] ClusterService cluster /api/aro_hcp/v1alpha1/clusters/2sslv6ma6mn5k5mt36luuom09289pq58 still exists (deletion dispatched at 2026-09-17T09:27:48Z); [clusterServiceStatus] ClusterService state is \\\"uninstalling\\\"; [hostedCluster] HostedCluster still exists; [descendantResources] remaining resources: 1 Microsoft.RedHatOpenShift/hcpOpenShiftClusters/nodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/nodepools/serviceProviderNodePools, 1 microsoft.redhatopenshift/hcpopenshiftclusters/serviceProviderClusters; [resourceTeardown] tearing down namespaces (2)\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"724c2a9602b01afbc546852f539ff27b105da0871a0374f448eae350e85e32ce","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f229c4ecc6ffd2e2f0f2e377dfb76a9b2fdb44ba5c3215dea5f0ce85ff49f862","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster deletion did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015da030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2vshj8kd4ngg\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"name\": \"2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:42:31.310516975Z\",\n      \"endTime\": \"2026-09-17T10:06:34.577233142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc0015da030\u003e: \n    failed to cleanup resource group: at least one hcp cluster failed to delete: failed waiting for hcpCluster=\"pause-bkp-cluster\" in resourcegroup=\"pause-bkp-e2e-2vshj8kd4ngg\" to finish deleting: GET https://rp.j0442496.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"name\": \"2b25208f-0e8c-46c2-96d1-92647ed090d2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T09:42:31.310516975Z\",\n      \"endTime\": \"2026-09-17T10:06:34.577233142Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster deletion did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100499155150442496","occurred_at":"2026-09-17T08:16:38Z","signature_id":"b8fb61f35748687f1492367050e150cb136907cb120fe4d9a22bd956f2ad9b80","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"27391593c89e1b6a3153b058ef31be166dd830240fb93e3caa733a62c02edc05","failure_pattern":"ERROR CODE: ResourceCollectionRequestsThrottled; provider Microsoft.Network","search_query":"ERROR CODE: ResourceCollectionRequestsThrottled","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:141]: failed to find private KAS internal LB IP in managed resource group \"private-kas-pzzdfb6jcgvd--managed\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001100480\u003e: \n    failed to list load balancers in \"private-kas-pzzdfb6jcgvd--managed\": GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-pzzdfb6jcgvd--managed/providers/Microsoft.Network/loadBalancers\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: ResourceCollectionRequestsThrottled\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceCollectionRequestsThrottled\",\n        \"message\": \"Operation 'Microsoft.Network/loadBalancers/read' failed as server encountered too many requests. Please try after '1' seconds. Tracking Id is '0bca3dc6-65f8-40d2-be6e-d4168ed4b62f'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:141]: failed to find private KAS internal LB IP in managed resource group \"private-kas-pzzdfb6jcgvd--managed\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001100480\u003e: \n    failed to list load balancers in \"private-kas-pzzdfb6jcgvd--managed\": GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-pzzdfb6jcgvd--managed/providers/Microsoft.Network/loadBalancers\n    --------------------------------------------------------------------------------\n    RESPONSE 429: 429 Too Many Requests\n    ERROR CODE: ResourceCollectionRequestsThrottled\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceCollectionRequestsThrottled\",\n        \"message\": \"Operation 'Microsoft.Network/loadBalancers/read' failed as server encountered too many requests. Please try after '1' seconds. Tracking Id is '0bca3dc6-65f8-40d2-be6e-d4168ed4b62f'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6856/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100488796448493568","occurred_at":"2026-09-17T07:35:28Z","signature_id":"dcc1d91a022ed0888281ddf9849c28aaaf8b8a73e6c45ef734d0a5c17a44b021","pr_number":6856,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"575b0de6701cb1cdcdd3b5c4e6523248471d3aa67c3c0c1fed243233e4014b33","failure_pattern":"ERROR CODE: ResourceGroupNotFound; provider Microsoft.Monitor","search_query":"ERROR CODE: ResourceGroupNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,4,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"AMA Metrics service metrics from SVC and MGMT clusters should be present in Azure Monitor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j4886144 in resource group hcp-underlay-ci01-j4886144\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b90a00\u003e: \n    failed to get workspace services-j4886144: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4886144/providers/Microsoft.Monitor/accounts/services-j4886144\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j4886144' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/ama_metrics.go:83]: failed to look up SVC Prometheus endpoint for workspace services-j4886144 in resource group hcp-underlay-ci01-j4886144\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001b90a00\u003e: \n    failed to get workspace services-j4886144: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4886144/providers/Microsoft.Monitor/accounts/services-j4886144\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceGroupNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceGroupNotFound\",\n        \"message\": \"Resource group 'hcp-underlay-ci01-j4886144' could not be found.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"1e24a52fcfa8fdc207f5da162e136847f4db22f73c90f28694d25d387bc9480d","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"657e53ad5d5680010d2301e3d7c635143dbc126a29582451d12bff3f530c8c1b","failure_pattern":"ERROR CODE: ResourceNotFound; detail message The resource 'locations/hcpOperationStatuses/\u003cuuid\u003e' under resource group '' was not found.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: ResourceNotFound","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rrwf84jdrq9t/cluster-hs-pre-dlzcc5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ce420\u003e: \n    failed to create HCP cluster cluster-hs-pre-dlzcc5: failed waiting for cluster=\"cluster-hs-pre-dlzcc5\" in resourcegroup=\"rg-hs-pre--rrwf84jdrq9t\" to finish creating: GET https://rp.j1442176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:82]: HCP cluster rg-hs-pre--rrwf84jdrq9t/cluster-hs-pre-dlzcc5 should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007ce420\u003e: \n    failed to create HCP cluster cluster-hs-pre-dlzcc5: failed waiting for cluster=\"cluster-hs-pre-dlzcc5\" in resourcegroup=\"rg-hs-pre--rrwf84jdrq9t\" to finish creating: GET https://rp.j1442176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\n    --------------------------------------------------------------------------------\n    RESPONSE 404: 404 Not Found\n    ERROR CODE: ResourceNotFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"ResourceNotFound\",\n        \"message\": \"The resource 'locations/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172' under resource group '' was not found.\",\n        \"target\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7d6063b8-a9f0-4a41-b701-82a552707172\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6939/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100735274731442176","occurred_at":"2026-09-17T23:54:53Z","signature_id":"a9eee6e574575c9254a566a1c15978b6cdd24af6e7e61c298d051c731edd627c","pr_number":6939,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a97a0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:195]: failed to upgrade node pool npupgrade-4-20 to version 4.21.33\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc0017a97a0\u003e: \n    failed waiting for nodepool=\"npupgrade-4-20\" in cluster=\"np-version-upgrade-cluster-plrmgv\" resourcegroup=\"rg-np-version-upgrade-plrmgv-2td2q96dfsp2\" to finish updating, caused by: timeout '45.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool npupgrade-4-20 in cluster np-version-upgrade-cluster-plrmgv in resource group rg-np-version-upgrade-plrmgv-2td2q96dfsp2, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100487527256297472","occurred_at":"2026-09-17T07:30:18Z","signature_id":"e98964207b5fd40111448e87aa9fa8936a8a788c4325943a9b61b7b5870f98ff","pr_number":6952,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7855bddb074e5ee4da59762c697f802a687000bfd9477e9f92fc724e190b9330","failure_pattern":"VerifyFIPSEnabled failed: failed to check FIPS mode on node \u003cnode\u003e: failed to get pod logs for pod fips-check-\u003cid\u003e in namespace e2e-fips-test-\u003cid\u003e on node \u003cnode\u003e after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get","search_query":"VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014ee660\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:123]: failed to verify FIPS is enabled on cluster fips-enabled-cluster\nUnexpected error:\n    \u003c*errors.joinError | 0xc0014ee660\u003e: \n    VerifyFIPSEnabled failed: failed to check FIPS mode on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv: failed to get pod logs for pod fips-check-rxfv5 in namespace e2e-fips-test-k5rlj on node h8s4t1i2l4b8s8g-np-1-tjxxr-g4lsv after retries: an error on the server (\"unknown\") has prevented the request from succeeding (get pods fips-check-rxfv5)\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6989/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569759195074560","occurred_at":"2026-09-17T12:57:11Z","signature_id":"b4cc553a2220034e450c979876c39e29fdc1278faa5ca56fae473d7a07ad2b7c","pr_number":6989,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,2,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T08:00:10Z\n  Ended:       2026-09-17T08:08:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9085312-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085312/providers/microsoft.monitor/accounts/hcps-j9085312\", namespace=\"ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"61e3547c-41a9-4b9c-82c4-b22d00b533e0\"\n  Description: Pod ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T08:00:11Z\n  Ended:       2026-09-17T08:24:10Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j9085312-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085312/providers/microsoft.monitor/accounts/hcps-j9085312\", namespace=\"ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f\", pod=\"etcd-0\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"61e3547c-41a9-4b9c-82c4-b22d00b533e0\"\n  Description: Pod ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f/etcd-0 (etcd) is in waiting state (reason: \"CrashLoopBackOff\")."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"62ae640dcc38eb25ad05e49560b842c80c55b99017dce7da9cea433518d78675","pr_number":6988,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,5,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T07:58:12Z\n  Ended:       2026-09-17T08:12:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9085312-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085312/providers/microsoft.monitor/accounts/hcps-j9085312\", namespace=\"ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-2qwp8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T07:59:15Z\n  Ended:       2026-09-17T08:27:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j9085312-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.31:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j9085312/providers/microsoft.monitor/accounts/hcps-j9085312\", namespace=\"ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-2qwp8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2ssklcv0l1s8075vhkc4vl3pjtknmrtp-y8w9l0f7i7g1m1f/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6988/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100478808179085312","occurred_at":"2026-09-17T06:55:45Z","signature_id":"78b86d170894f1306383271eb64e5c7231bc9db1d4dade7d27fb2bd571aeb710","pr_number":6988,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,3,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T23:37:25Z\n  Ended:       2026-09-17T23:53:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9695872-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-17T23:46:22Z\n  Ended:       2026-09-17T23:56:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j9695872-svc\", component=\"backend\", name=\"csstatedump\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue csstatedump has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6964/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100712898639695872","occurred_at":"2026-09-17T22:25:59Z","signature_id":"3b128919de15d094b3050816e84bfdc2f25e06d586dd063b36f0c63378fdbfef","pr_number":6964,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-17T21:40:09Z\n  Ended:       2026-09-17T21:58:09Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j1728768-svc\", component=\"backend\", name=\"externalauthclusterservicecreate\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue externalauthclusterservicecreate has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6999/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100680641501728768","occurred_at":"2026-09-17T20:17:48Z","signature_id":"f3ebbdf3ff2dce8e5f0f8347ed14b1024c81d64efecc73041c83a810441bae40","pr_number":6999,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T13:42:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j4979456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4979456/providers/microsoft.monitor/accounts/services-j4979456\", namespace=\"arobit\", pod=\"arobit-forwarder-k26lv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9061149b-822a-4131-9e5f-b554ee1870de\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-k26lv on cluster ci01-j4979456-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T13:42:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j4979456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"fluentbit\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.70:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4979456/providers/microsoft.monitor/accounts/services-j4979456\", namespace=\"arobit\", pod=\"arobit-forwarder-k26lv\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9061149b-822a-4131-9e5f-b554ee1870de\"\n  Description: Container fluentbit in pod arobit/arobit-forwarder-k26lv on cluster ci01-j4979456-mgmt-2 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"6db51492a3864e98312b98dbacda80d9a85398c3e3f34314664719d56a168f65","pr_number":6756,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:39:34Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:40:35Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-17T02:46:35Z\n  Ended:       2026-09-17T02:54:36Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-17T02:46:35Z\n  Ended:       2026-09-17T02:59:37Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.14:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7881088/providers/microsoft.monitor/accounts/services-j7881088\", namespace=\"klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7\", pod=\"arohcp-monitor-kube-state-metrics-574f657d96-rcwzm\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2ssfo2opt0bd9ml6dnilop3d3i40c0r7/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"4d675a2aa4e866465ba35a460544e188e20719f62123eb05ebda79f62db3498b","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,5,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T02:04:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7881088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-6f59c7c94f-hsk8t\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-6f59c7c94f-hsk8t has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6940/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100388490737881088","occurred_at":"2026-09-17T00:56:54Z","signature_id":"9a8fc1c68aeeef65c0fa2d8774b771f9f44e0b8d2622880bad833fdcea513992","pr_number":6940,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-17T21:40:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j0025600-mgmt-2\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6945/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100689397040025600","occurred_at":"2026-09-17T20:52:35Z","signature_id":"c8d2c94c46ea41bb6616321fc23a840d9b2ec094985319abefcb6af50ae87597","pr_number":6945,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"bac1c5cb8cd2257dc73ed97f502419b2bb0c5622fe0e3311e3284099d74a2633","failure_pattern":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"\u003curl\u003e","search_query":"clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,2,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000c05990\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53121-\u003e172.30.0.10:53: read: connection refused\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:356]: Timed out after 300.007s.\nClusterIssuer should become Ready (ACME account registered)\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000c05990\u003e: \n    clusterIssuer not ready: reason=ErrRegisterACMEAccount message=Failed to register ACME account: Get \"https://acme-staging-v02.api.letsencrypt.org/directory\": dial tcp: lookup acme-staging-v02.api.letsencrypt.org on 172.30.0.10:53: read udp 10.128.0.10:53121-\u003e172.30.0.10:53: read: connection refused\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/batch/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100561361296691200","occurred_at":"2026-09-17T12:23:47Z","signature_id":"41e17c7a4eecb6c1ed4ee23560b530d6b6a42361cc6e3ef0250dbdef28f6d545","pr_number":6933,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"d33373d20e5996675bec12509adcc946f8d57b13e0f18cc57679d39e08115eb4","failure_pattern":"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"\u003cresource-group\u003e\": Get \"\u003curl\u003e\": stream error: stream ID 13; CANCEL; received from peer","search_query":"failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d270f8\u003e: \n    failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:295]: Unexpected error:\n    \u003c*errors.joinError | 0xc000d270f8\u003e: \n    failed to cleanup resource group: failed listing hcp clusters in resourcegroup=\"rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8\": Get \"https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/rg-np-version-upgrade-pr4qr2-pmcbs2hlnrf8?api-version=2021-04-01\": stream error: stream ID 13; CANCEL; received from peer\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6157/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100569681449455616","occurred_at":"2026-09-17T12:56:50Z","signature_id":"2126196a18f3d500d8a52360e541d2fceb67e8a67c46b2636cf54189a72bfe35","pr_number":6157,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0009e8ec0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"name\": \"f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:19.756903274Z\",\n      \"endTime\": \"2026-09-17T13:12:54.770011764Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:127]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc0009e8ec0\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-qbcpd5p47nxj\" to finish creating: GET https://rp.j4979456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"name\": \"f3bad1b6-65e5-4da7-a96f-1c7006af3c80\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T13:12:19.756903274Z\",\n      \"endTime\": \"2026-09-17T13:12:54.770011764Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100554708664979456","occurred_at":"2026-09-17T11:57:12Z","signature_id":"7dda8cc2984aa1b8ad7daf00d042d0064ecf3188c6f0d908489ee4571d215e40","pr_number":6756,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"40d94d1862472748004cc33c4c935e1c5142aa767d3a1a077dd2606d1dc7c7d3","failure_pattern":"failed to roll out Helm release: Internal error occurred: failed calling webhook \"validation.istio.io\": failed to call webhook: Post \"\u003curl\u003e\": EOF","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SessionGate/service/deploy","occurrences":1}],"full_error_samples":["time=2026-09-17T09:16:31.171Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-17T09:16:31.636Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy values=\"map[affinity:map[] auth:map[audience:6dae42f8-4368-4678-94ff-3960e28e3630 authority:https://sts.windows.net/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] azure:map[clientId:53204456-a0a3-4fd2-89f7-36eb569bbb5d cloudEnvironmentName:AzurePublicCloud tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] credsKeyVault:map[name:aro-hcp-dev-svc-kv secret:sessiongate-cert-ci01-j2597888] deployment:map[replicas:2 requests:map[cpu:250m memory:512Mi]] httpRoute:map[gateway:map[name:ops-ingress-gateway namespace:aks-istio-ingress sectionName:sessiongate-https] host:sessiongate.j2597888.hcpsvc.osadev.cloud] image:map[digest:sha256:dfb4a580c4ae075df76e82458246c28e24407e254922d6c888d0d2793d3c1737 registry:arohcpsvcdev.azurecr.io repository:ci-op-6i9x21qi/pipeline] leaderElection:map[leaseDuration:15s renewDeadline:10s retryPeriod:2s] logLevel:2 mise:map[deploy:false policyLabel:Session Gate] nodeSelector:map[] pullBinding:map[registry:arohcpsvcdev.azurecr.io scope:repository:ci-op-6i9x21qi/pipeline:pull workloadIdentityClientId:0ee3f5a0-d80b-4385-abcc-91385fcf6f70 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] service:map[name:sessiongate port:8080] serviceAccount:map[name:sessiongate workloadIdentityClientId:15494d5f-cf01-4a47-ae22-344126602800 workloadIdentityTenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX] tolerations:[]]\"\ntime=2026-09-17T09:16:31.636Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:31.786Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy namespace=sessiongate\ntime=2026-09-17T09:16:31.977Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy dryRun=false\ntime=2026-09-17T09:16:31.977Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:32.031Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy\ntime=2026-09-17T09:16:37.667Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\"\ntime=2026-09-17T09:16:37.667Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SessionGate resourceGroup=service step=deploy err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: Internal error occurred: failed calling webhook \\\"validation.istio.io\\\": failed to call webhook: Post \\\"https://istiod-asm-1-29.aks-istio-system.svc:443/validate?timeout=10s\\\": EOF\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6990/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100504522802597888","occurred_at":"2026-09-17T08:37:58Z","signature_id":"bbb80a3447e86b156cad4ed50a4810d3fa341869a205ecefaab92d1964bbcc6f","pr_number":6990,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"35fcc68c27835aae14daafdef12fc813fb2e38cc3e35d2f974eb6cc333b3ce7e","failure_pattern":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane","search_query":"kube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:85]: Timed out after 900.000s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:83 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:85]: Timed out after 900.000s.\nkube_node_info metrics never appeared in Azure Monitor for any hostedcontrolplane\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:83 with:\nexpected kube_node_info metrics for at least one hostedcontrolplane but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100556682735128576","occurred_at":"2026-09-17T12:05:14Z","signature_id":"7be65470254fc4c585209a09735c416c75334c6ead21b5d5328120538ae66b13","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"cea54d0df4cc4df84a4bbaf41795235750a198ce743843fbdff1373606efa1aa","failure_pattern":"kube_node_info metrics never appeared in Azure Monitor for cluster \"\"","search_query":"kube_node_info metrics never appeared in Azure Monitor for cluster \"\"","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics kube_node_info metrics should be present in Azure Monitor for the happy-path cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:97]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for cluster \"\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:95 with:\nexpected kube_node_info metrics for cluster \"\" but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:97]: Timed out after 900.001s.\nkube_node_info metrics never appeared in Azure Monitor for cluster \"\"\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:95 with:\nexpected kube_node_info metrics for cluster \"\" but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6672/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100517137754886144","occurred_at":"2026-09-17T09:28:04Z","signature_id":"4645b1f607eafec38a4f58b597cf9ee778993ba4ba3bc944eda97fc9f0f70da7","pr_number":6672,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"50012cc84060fc59db5e840f9933bf1a542baf6598061d9d1e54da53aa955055","failure_pattern":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","search_query":"permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,2,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to log into a cluster via a breakglass session","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.086s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001018980\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiregistration.k8s.io/apiservices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusterversions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusteroperators: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/infrastructures: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list security.openshift.io/securitycontextconstraints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list route.openshift.io/routes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list image.openshift.io/imagestreams: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:480]: Timed out after 34.086s.\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001018980\u003e: \n    permission checks failed: SSAR failed for list /services: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /endpoints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /events: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /persistentvolumeclaims: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /persistentvolumes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /serviceaccounts: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /resourcequotas: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /limitranges: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list /replicationcontrollers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /componentstatuses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for get /pods/log: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/deployments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/daemonsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/statefulsets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apps/replicasets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/jobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list batch/cronjobs: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/networkpolicies: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list networking.k8s.io/ingresses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterroles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/clusterrolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/roles: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list rbac.authorization.k8s.io/rolebindings: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/storageclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list storage.k8s.io/volumeattachments: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiextensions.k8s.io/customresourcedefinitions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list policy/poddisruptionbudgets: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list autoscaling/horizontalpodautoscalers: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list coordination.k8s.io/leases: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list discovery.k8s.io/endpointslices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list certificates.k8s.io/certificatesigningrequests: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list scheduling.k8s.io/priorityclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list node.k8s.io/runtimeclasses: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/mutatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list admissionregistration.k8s.io/validatingwebhookconfigurations: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list apiregistration.k8s.io/apiservices: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusterversions: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/clusteroperators: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list config.openshift.io/infrastructures: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list security.openshift.io/securitycontextconstraints: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list route.openshift.io/routes: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    SSAR failed for list image.openshift.io/imagestreams: the server could not find the requested resource (post selfsubjectaccessreviews.authorization.k8s.io)\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100495749199433728","occurred_at":"2026-09-17T08:03:06Z","signature_id":"d145278289b57df8d05b7f53866f25f2a3e6803008a36d7c5b0f96179504086e","pr_number":6364,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"62df4554fa8988de83bcea5588b7f5f69715e1db3483f50b3137f97138b59e77","failure_pattern":"run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"\u003cvm\u003e\" (output: \"000\")","search_query":"run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.5102040816326531,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 654.956s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00277e080\u003e: \n    run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:166]: Timed out after 654.956s.\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:157 with:\nRunVMCommand failed for ingress connectivity test\nUnexpected error:\n    \u003c*errors.errorString | 0xc00277e080\u003e: \n    run command \"e2e-runcommand-1789636815233707722\" failed with exit code 28 on VM \"private-ingress-test-vm\" (output: \"000\")\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2100494074514509824","occurred_at":"2026-09-17T07:56:27Z","signature_id":"1395a5c67402b5879cd22d694e49480a23894641a58ba386a295937a83eff76f","pr_number":6752,"after_last_push_of_merged_pr":true}]}]},{"environment":"int","summary":{"total_runs":8,"failed_runs":5,"raw_occurrences":6,"matched_occurrences":2,"runs_affected":1},"rows":[{"environment":"int","failure_pattern_id":"44813a24efc9a4e87de23a1d03f218de3f44f990811139bedf70528921d1e908","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:162]: failed to verify HCP cluster ephemeral-disk is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017886a8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:162]: failed to verify HCP cluster ephemeral-disk is viable\nUnexpected error:\n    \u003c*errors.joinError | 0xc0017886a8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ephemeral-disk.foc3.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:49368-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100532556209328128","occurred_at":"2026-09-17T10:29:22Z","signature_id":"a7b733023a299f40faa1bcb49b8265ed28d3f85ddfbf335757885b57a1e7d20f","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"int","failure_pattern_id":"221cb0b46f3bef91cff2ee06c068eca8fb3646ef7eeb7ed9cd39ae59c1446e81","failure_pattern":"list nodes (nodePool=\u003cnodepool\u003e): Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:612]: failed to get node release images before downgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019a0220\u003e: \n    list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:612]: failed to get node release images before downgrade\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0019a0220\u003e: \n    list nodes (nodePool=npdg-4-21): Get \"https://api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net:443/api/v1/nodes\": dial tcp: lookup api.a4m2z1c5e2c5i9i.2o87.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.130.98.17:59019-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2100532556209328128","occurred_at":"2026-09-17T10:29:22Z","signature_id":"06051e7fd69c31d191475d70ad084a2f5f338204d9dcb2ccb5a0566c0fc34ac7","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"raw_occurrences":18,"matched_occurrences":16,"runs_affected":3},"rows":[{"environment":"prod","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":5,"runs_affected":3,"failed_runs":3,"run_impact_percent":60,"anchor_occurrences":5,"after_last_push_count":6,"also_in":["DEV","STG"],"trend_counts":[0,0,0,0,0,1,5],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062e0a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-khvwxc7hm457\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9a805e6d-f4c1-448f-ac8d-eb4629159049\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9a805e6d-f4c1-448f-ac8d-eb4629159049\",\n      \"name\": \"9a805e6d-f4c1-448f-ac8d-eb4629159049\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:14:40.679038392Z\",\n      \"endTime\": \"2026-09-17T23:33:46.186414821Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:222]: failed to create nodepool \"np-1\" for authorized CIDRs cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00062e0a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-khvwxc7hm457\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9a805e6d-f4c1-448f-ac8d-eb4629159049\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/9a805e6d-f4c1-448f-ac8d-eb4629159049\",\n      \"name\": \"9a805e6d-f4c1-448f-ac8d-eb4629159049\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T23:14:40.679038392Z\",\n      \"endTime\": \"2026-09-17T23:33:46.186414821Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002740c00\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5ddg8tcqt24f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a27ca6a0-c950-4180-9609-7a9ade0dfaf2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a27ca6a0-c950-4180-9609-7a9ade0dfaf2\",\n      \"name\": \"a27ca6a0-c950-4180-9609-7a9ade0dfaf2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T22:32:08.724911504Z\",\n      \"endTime\": \"2026-09-17T22:51:12.585683878Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc002740c00\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-5ddg8tcqt24f\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a27ca6a0-c950-4180-9609-7a9ade0dfaf2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/a27ca6a0-c950-4180-9609-7a9ade0dfaf2\",\n      \"name\": \"a27ca6a0-c950-4180-9609-7a9ade0dfaf2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T22:32:08.724911504Z\",\n      \"endTime\": \"2026-09-17T22:51:12.585683878Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a4edc0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-rrzr5s\" in resourcegroup=\"rg-np-noedge-rrzr5s-tw6bdclngxdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e383065-95e4-447a-8c07-f8b7ac95870b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e383065-95e4-447a-8c07-f8b7ac95870b\",\n      \"name\": \"4e383065-95e4-447a-8c07-f8b7ac95870b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T22:45:09.171165434Z\",\n      \"endTime\": \"2026-09-17T23:04:12.617623397Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.37\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a4edc0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-rrzr5s\" in resourcegroup=\"rg-np-noedge-rrzr5s-tw6bdclngxdp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e383065-95e4-447a-8c07-f8b7ac95870b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e383065-95e4-447a-8c07-f8b7ac95870b\",\n      \"name\": \"4e383065-95e4-447a-8c07-f8b7ac95870b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T22:45:09.171165434Z\",\n      \"endTime\": \"2026-09-17T23:04:12.617623397Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"8af91b6a912b93a55a51953f18f51b6e88bc18db738006d3b71d9e5d820486c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","signature_id":"be41217a9374881e99f1a9416ccadf32d5a5da3fe931b3a6439f304be981164d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","signature_id":"ce783312029a03f3c6f81cf899cc96d5f27f953623a9a5fa1b6b579e78455f50","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2100694048263114752","occurred_at":"2026-09-17T21:11:04Z","signature_id":"e9d197f55b7c8094b6f94fd2ac343ddfe7d54231e67ed0c0282b273041aea3b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2100404440946184192","occurred_at":"2026-09-17T02:00:16Z","signature_id":"ce1e7db080deea06cb7610e1f4b4bf939ebd4afb01e8c1a7763c9d9cd1a9bd32","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":4,"after_last_push_count":9,"trend_counts":[0,0,0,0,0,5,4],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0020c7110\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-9jm8bqptv2wt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0020c7110\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-9jm8bqptv2wt/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002b70300\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-9mx94tspcdqp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:274]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc002b70300\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-9mx94tspcdqp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013e8bd0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-n8nkjmlflmx2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013e8bd0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-n8nkjmlflmx2/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"19d9edd34f8e48701cd8bbda924191e80690cc1893c6e47aa7458abc045a4095","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"ad4219e21e3f54d13de0b5a81134597e91fe66a1442cf0f6968126d652406ccf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"b64e2e209238a4291f7bc6ca875c4002b2b72ce61c22b00cc341eff1d3d7d329","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"d144fcda6ced4631fcc19c7abd504ffe793c4fffbba643dd7b8a0d80f55c67e1","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000996700\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r7m2fk\" in resourcegroup=\"rg-np-version-upgrade-r7m2fk-5jlmq454vn2n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7m2fk-5jlmq454vn2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r7m2fk/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000996700\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-r7m2fk\" in resourcegroup=\"rg-np-version-upgrade-r7m2fk-5jlmq454vn2n\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-r7m2fk-5jlmq454vn2n/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-r7m2fk/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001466840\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-psjtgg\" resourcegroup=\"rg-np-dg-minor-psjtgg-2t4zvgfl9nx7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-psjtgg-2t4zvgfl9nx7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-psjtgg/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc001466840\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-psjtgg\" resourcegroup=\"rg-np-dg-minor-psjtgg-2t4zvgfl9nx7\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-psjtgg-2t4zvgfl9nx7/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-psjtgg/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007184c0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-cqdwqx\" in resourcegroup=\"rg-np-skip-minor-cqdwqx-w27zwxzrpszc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-cqdwqx-w27zwxzrpszc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-cqdwqx/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-16-210648\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0007184c0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-cqdwqx\" in resourcegroup=\"rg-np-skip-minor-cqdwqx-w27zwxzrpszc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-cqdwqx-w27zwxzrpszc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-cqdwqx/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-16-210648\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"475d18438d6129ee057302e14cb4929cac287683cc42b505124692ed9000396d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"5b8b71dd55429c7ea7e12ce3e8d437a79ae271abc6b3fd1cf66b2d542a2f54ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"9f76555451648e830890a7c9cd43251ffcd8ea938c9675715ab08891b03af525","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"ae583f4fcd37f743d2e4acb91167ea667e50e9c13e01db10c235ffe6dd99caa7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d6164cff5eccd3db4fabd19fbe08aa12f39c6b3d5c2202182c9f291029d16282","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.22\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00079eb40\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-8mb2dp88brbx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-8mb2dp88brbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:90]: failed to create HCP cluster \"private-kas\" with private KAS\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00079eb40\u003e: \n    failed to create HCP cluster private-kas: failed starting cluster creation \"private-kas\" in resourcegroup=\"private-kas-8mb2dp88brbx\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-kas-8mb2dp88brbx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-kas\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.22\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"eedd7ed28a2e498144c6499476b15f9b62b521f665937a7144186b5981a0ab86","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"702b4093e9521ad672ccffd800bca261603e5674f6962331f6eb7d30c5570cc4","failure_pattern":"ERROR CODE: SubscriptionNotRegistered; provider Microsoft.Insights","search_query":"ERROR CODE: SubscriptionNotRegistered","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cv4zb86lrk99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f21140\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cv4zb86lrk99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:319]: failed to create storage account diagnostic setting for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cv4zb86lrk99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc000f21140\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-cv4zb86lrk99/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster/providers/Microsoft.Insights/diagnosticSettings/shoebox-storage-diag\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: SubscriptionNotRegistered\n    --------------------------------------------------------------------------------\n    {\n      \"code\": \"SubscriptionNotRegistered\",\n      \"message\": \"The subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' is not registered to use microsoft.insights.\"\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"a0e7d58faa2918f496570ed9c5f40cc4898c7cc2d361ef4097b193762f74fdcc","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,0,1,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fe8ae0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.009s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000fe8ae0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-16-151028, targetMinor=5.1.0-0.nightly-multi-2026-09-15-135250) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-09-15-135250\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2100721533713387520","occurred_at":"2026-09-17T23:00:17Z","signature_id":"787555a75756b0bcd3c50e5022f58b6e6c990b64d6f87d9979ae97767e7916f5","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":4,"failed_runs":3,"raw_occurrences":5,"matched_occurrences":4,"runs_affected":2},"rows":[{"environment":"stg","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":3,"after_last_push_count":4,"also_in":["DEV","PROD"],"trend_counts":[0,0,0,0,0,1,3],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc2640\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xfvdn4xsffw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"name\": \"98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:40:34.02322608Z\",\n      \"endTime\": \"2026-09-17T05:59:42.272250066Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_fips_mode.go:103]: failed to create node pool \"np-1\" for fips-enabled cluster \"fips-enabled-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fc2640\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"fips-enabled-cluster\" in resourcegroup=\"fips-enabled-4xfvdn4xsffw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"name\": \"98b08bb9-0eee-4840-9535-91464d8458a2\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:40:34.02322608Z\",\n      \"endTime\": \"2026-09-17T05:59:42.272250066Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0035265a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-rcr2wwpjvcjv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"name\": \"5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:39:36.012772677Z\",\n      \"endTime\": \"2026-09-17T05:58:42.168749593Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delete_cx_rg.go:139]: failed to create node pools in parallel for cluster \"cx-rg-hcp-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0035265a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"cx-rg-hcp-cluster\" in resourcegroup=\"cx-rg-cluster-rcr2wwpjvcjv\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"name\": \"5b397553-ee61-4674-b26e-e3cf8ae2dc9d\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T05:39:36.012772677Z\",\n      \"endTime\": \"2026-09-17T05:58:42.168749593Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-72xhwq\" for cluster \"cluster-candidate-4-22-72xhwq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009bb400\u003e: \n    failed to create NodePool np-72xhwq: failed waiting for nodepool=\"np-72xhwq\" for cluster \"cluster-candidate-4-22-72xhwq\" in resourcegroup=\"rg-candidate-4-22-ffxlnxmpk4tb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ee3e506-9091-4aa0-ad2c-045d3e78731e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ee3e506-9091-4aa0-ad2c-045d3e78731e\",\n      \"name\": \"8ee3e506-9091-4aa0-ad2c-045d3e78731e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T06:05:40.036081638Z\",\n      \"endTime\": \"2026-09-17T06:24:44.492140979Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:175]: failed to create node pool \"np-72xhwq\" for cluster \"cluster-candidate-4-22-72xhwq\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0009bb400\u003e: \n    failed to create NodePool np-72xhwq: failed waiting for nodepool=\"np-72xhwq\" for cluster \"cluster-candidate-4-22-72xhwq\" in resourcegroup=\"rg-candidate-4-22-ffxlnxmpk4tb\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ee3e506-9091-4aa0-ad2c-045d3e78731e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8ee3e506-9091-4aa0-ad2c-045d3e78731e\",\n      \"name\": \"8ee3e506-9091-4aa0-ad2c-045d3e78731e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-17T06:05:40.036081638Z\",\n      \"endTime\": \"2026-09-17T06:24:44.492140979Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","signature_id":"573bf63f25f718e6e10d14bc2ae7830ca909cb9577c72529929dab5db0a4d7ee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","signature_id":"cc190a2c1395e4f5ad1b156f9318c22909f9bd03dbf07b404cb1ad51d8dfd21e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100452501269516288","occurred_at":"2026-09-17T05:11:15Z","signature_id":"d97b1d417fbed487ffa515ee9e94f38afd2685c1f5cc55df9fad23cc5afc6bc9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"9f24424c672066cba3c7cb45f333ebe45b01b356fab007f5a65bee0f062d44cf","failure_pattern":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: read udp \u003cip\u003e-\u003e\u003cip\u003e: i/o timeout","search_query":"VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":25,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["INT"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-11..2026-09-17 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-4-22-j5vkzr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015d2858\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:148]: failed to verify cluster \"cp-ystream-upgrade-4-22-j5vkzr\" is viable before upgrade\nUnexpected error:\n    \u003c*errors.joinError | 0xc0015d2858\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/api/v1/namespaces/default/services\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.z1t1t5s4i6u3r6h.s9td.uksouthstaging.aroapp-hcp.io on 172.30.0.10:53: read udp 10.130.98.22:39829-\u003e172.30.0.10:53: i/o timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2100534031866138624","occurred_at":"2026-09-17T10:35:13Z","signature_id":"aac82bf00f815d526a50ddb15d7fc450980f914b55421ea2243e84387908cb3a","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}